6 * This file can be included into eBPF kernel programs. It contains
7 * a couple of useful helper functions, map/section ABI (bpf_elf.h),
8 * misc macros and some eBPF specific LLVM built-ins.
13 #include <linux/pkt_cls.h>
14 #include <linux/bpf.h>
15 #include <linux/filter.h>
17 #include <asm/byteorder.h>
24 # define __stringify(X) #X
27 #ifndef __maybe_unused
28 # define __maybe_unused __attribute__((__unused__))
32 # define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
36 # define likely(X) __builtin_expect(!!(X), 1)
40 # define unlikely(X) __builtin_expect(!!(X), 0)
44 # define htons(X) __constant_htons((X))
48 # define ntohs(X) __constant_ntohs((X))
52 # define htonl(X) __constant_htonl((X))
56 # define ntohl(X) __constant_ntohl((X))
60 # define __inline__ __attribute__((always_inline))
63 /** Section helper macros. */
66 # define __section(NAME) \
67 __attribute__((section(NAME), used))
70 #ifndef __section_tail
71 # define __section_tail(ID, KEY) \
72 __section(__stringify(ID) "/" __stringify(KEY))
75 #ifndef __section_cls_entry
76 # define __section_cls_entry \
77 __section(ELF_SECTION_CLASSIFIER)
80 #ifndef __section_act_entry
81 # define __section_act_entry \
82 __section(ELF_SECTION_ACTION)
85 #ifndef __section_license
86 # define __section_license \
87 __section(ELF_SECTION_LICENSE)
90 #ifndef __section_maps
91 # define __section_maps \
92 __section(ELF_SECTION_MAPS)
95 /** Declaration helper macros. */
98 # define BPF_LICENSE(NAME) \
99 char ____license[] __section_license = NAME
102 /** Classifier helper */
104 #ifndef BPF_H_DEFAULT
105 # define BPF_H_DEFAULT -1
108 /** BPF helper functions for tc. Individual flags are in linux/bpf.h */
111 # define __BPF_FUNC(NAME, ...) \
112 (* NAME)(__VA_ARGS__) __maybe_unused
116 # define BPF_FUNC(NAME, ...) \
117 __BPF_FUNC(NAME, __VA_ARGS__) = (void *) BPF_FUNC_##NAME
120 /* Map access/manipulation */
121 static void *BPF_FUNC(map_lookup_elem
, void *map
, const void *key
);
122 static int BPF_FUNC(map_update_elem
, void *map
, const void *key
,
123 const void *value
, uint32_t flags
);
124 static int BPF_FUNC(map_delete_elem
, void *map
, const void *key
);
127 static uint64_t BPF_FUNC(ktime_get_ns
);
131 /* FIXME: __attribute__ ((format(printf, 1, 3))) not possible unless
132 * llvm bug https://llvm.org/bugs/show_bug.cgi?id=26243 gets resolved.
133 * It would require ____fmt to be made const, which generates a reloc
136 static void BPF_FUNC(trace_printk
, const char *fmt
, int fmt_size
, ...);
139 # define printt(fmt, ...) \
141 char ____fmt[] = fmt; \
142 trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
147 static uint32_t BPF_FUNC(get_prandom_u32
);
150 static void BPF_FUNC(tail_call
, struct __sk_buff
*skb
, void *map
,
154 static uint32_t BPF_FUNC(get_smp_processor_id
);
155 static uint32_t BPF_FUNC(get_numa_node_id
);
157 /* Packet misc meta data */
158 static uint32_t BPF_FUNC(get_cgroup_classid
, struct __sk_buff
*skb
);
159 static int BPF_FUNC(skb_under_cgroup
, void *map
, uint32_t index
);
161 static uint32_t BPF_FUNC(get_route_realm
, struct __sk_buff
*skb
);
162 static uint32_t BPF_FUNC(get_hash_recalc
, struct __sk_buff
*skb
);
163 static uint32_t BPF_FUNC(set_hash_invalid
, struct __sk_buff
*skb
);
165 /* Packet redirection */
166 static int BPF_FUNC(redirect
, int ifindex
, uint32_t flags
);
167 static int BPF_FUNC(clone_redirect
, struct __sk_buff
*skb
, int ifindex
,
170 /* Packet manipulation */
171 static int BPF_FUNC(skb_load_bytes
, struct __sk_buff
*skb
, uint32_t off
,
172 void *to
, uint32_t len
);
173 static int BPF_FUNC(skb_store_bytes
, struct __sk_buff
*skb
, uint32_t off
,
174 const void *from
, uint32_t len
, uint32_t flags
);
176 static int BPF_FUNC(l3_csum_replace
, struct __sk_buff
*skb
, uint32_t off
,
177 uint32_t from
, uint32_t to
, uint32_t flags
);
178 static int BPF_FUNC(l4_csum_replace
, struct __sk_buff
*skb
, uint32_t off
,
179 uint32_t from
, uint32_t to
, uint32_t flags
);
180 static int BPF_FUNC(csum_diff
, const void *from
, uint32_t from_size
,
181 const void *to
, uint32_t to_size
, uint32_t seed
);
182 static int BPF_FUNC(csum_update
, struct __sk_buff
*skb
, uint32_t wsum
);
184 static int BPF_FUNC(skb_change_type
, struct __sk_buff
*skb
, uint32_t type
);
185 static int BPF_FUNC(skb_change_proto
, struct __sk_buff
*skb
, uint32_t proto
,
187 static int BPF_FUNC(skb_change_tail
, struct __sk_buff
*skb
, uint32_t nlen
,
190 static int BPF_FUNC(skb_pull_data
, struct __sk_buff
*skb
, uint32_t len
);
192 /* Event notification */
193 static int __BPF_FUNC(skb_event_output
, struct __sk_buff
*skb
, void *map
,
194 uint64_t index
, const void *data
, uint32_t size
) =
195 (void *) BPF_FUNC_perf_event_output
;
197 /* Packet vlan encap/decap */
198 static int BPF_FUNC(skb_vlan_push
, struct __sk_buff
*skb
, uint16_t proto
,
200 static int BPF_FUNC(skb_vlan_pop
, struct __sk_buff
*skb
);
202 /* Packet tunnel encap/decap */
203 static int BPF_FUNC(skb_get_tunnel_key
, struct __sk_buff
*skb
,
204 struct bpf_tunnel_key
*to
, uint32_t size
, uint32_t flags
);
205 static int BPF_FUNC(skb_set_tunnel_key
, struct __sk_buff
*skb
,
206 const struct bpf_tunnel_key
*from
, uint32_t size
,
209 static int BPF_FUNC(skb_get_tunnel_opt
, struct __sk_buff
*skb
,
210 void *to
, uint32_t size
);
211 static int BPF_FUNC(skb_set_tunnel_opt
, struct __sk_buff
*skb
,
212 const void *from
, uint32_t size
);
214 /** LLVM built-ins, mem*() routines work for constant size */
217 # define lock_xadd(ptr, val) ((void) __sync_fetch_and_add(ptr, val))
221 # define memset(s, c, n) __builtin_memset((s), (c), (n))
225 # define memcpy(d, s, n) __builtin_memcpy((d), (s), (n))
229 # define memmove(d, s, n) __builtin_memmove((d), (s), (n))
232 /* FIXME: __builtin_memcmp() is not yet fully useable unless llvm bug
233 * https://llvm.org/bugs/show_bug.cgi?id=26218 gets resolved. Also
234 * this one would generate a reloc entry (non-map), otherwise.
238 # define memcmp(a, b, n) __builtin_memcmp((a), (b), (n))
242 unsigned long long load_byte(void *skb
, unsigned long long off
)
243 asm ("llvm.bpf.load.byte");
245 unsigned long long load_half(void *skb
, unsigned long long off
)
246 asm ("llvm.bpf.load.half");
248 unsigned long long load_word(void *skb
, unsigned long long off
)
249 asm ("llvm.bpf.load.word");
251 #endif /* __BPF_API__ */