6 * This file can be included into eBPF kernel programs. It contains
7 * a couple of useful helper functions, map/section ABI (bpf_elf.h),
8 * misc macros and some eBPF specific LLVM built-ins.
13 #include <linux/pkt_cls.h>
14 #include <linux/bpf.h>
15 #include <linux/filter.h>
17 #include <asm/byteorder.h>
24 # define __stringify(X) #X
27 #ifndef __maybe_unused
28 # define __maybe_unused __attribute__((__unused__))
32 # define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
36 # define likely(X) __builtin_expect(!!(X), 1)
40 # define unlikely(X) __builtin_expect(!!(X), 0)
44 # define htons(X) __constant_htons((X))
48 # define ntohs(X) __constant_ntohs((X))
52 # define htonl(X) __constant_htonl((X))
56 # define ntohl(X) __constant_ntohl((X))
60 # define __inline__ __attribute__((always_inline))
63 /** Section helper macros. */
66 # define __section(NAME) \
67 __attribute__((section(NAME), used))
70 #ifndef __section_tail
71 # define __section_tail(ID, KEY) \
72 __section(__stringify(ID) "/" __stringify(KEY))
75 #ifndef __section_xdp_entry
76 # define __section_xdp_entry \
77 __section(ELF_SECTION_PROG)
80 #ifndef __section_cls_entry
81 # define __section_cls_entry \
82 __section(ELF_SECTION_CLASSIFIER)
85 #ifndef __section_act_entry
86 # define __section_act_entry \
87 __section(ELF_SECTION_ACTION)
90 #ifndef __section_lwt_entry
91 # define __section_lwt_entry \
92 __section(ELF_SECTION_PROG)
95 #ifndef __section_license
96 # define __section_license \
97 __section(ELF_SECTION_LICENSE)
100 #ifndef __section_maps
101 # define __section_maps \
102 __section(ELF_SECTION_MAPS)
105 /** Declaration helper macros. */
108 # define BPF_LICENSE(NAME) \
109 char ____license[] __section_license = NAME
112 /** Classifier helper */
114 #ifndef BPF_H_DEFAULT
115 # define BPF_H_DEFAULT -1
118 /** BPF helper functions for tc. Individual flags are in linux/bpf.h */
121 # define __BPF_FUNC(NAME, ...) \
122 (* NAME)(__VA_ARGS__) __maybe_unused
126 # define BPF_FUNC(NAME, ...) \
127 __BPF_FUNC(NAME, __VA_ARGS__) = (void *) BPF_FUNC_##NAME
130 /* Map access/manipulation */
131 static void *BPF_FUNC(map_lookup_elem
, void *map
, const void *key
);
132 static int BPF_FUNC(map_update_elem
, void *map
, const void *key
,
133 const void *value
, uint32_t flags
);
134 static int BPF_FUNC(map_delete_elem
, void *map
, const void *key
);
137 static uint64_t BPF_FUNC(ktime_get_ns
);
141 /* FIXME: __attribute__ ((format(printf, 1, 3))) not possible unless
142 * llvm bug https://llvm.org/bugs/show_bug.cgi?id=26243 gets resolved.
143 * It would require ____fmt to be made const, which generates a reloc
146 static void BPF_FUNC(trace_printk
, const char *fmt
, int fmt_size
, ...);
149 # define printt(fmt, ...) \
151 char ____fmt[] = fmt; \
152 trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
157 static uint32_t BPF_FUNC(get_prandom_u32
);
160 static void BPF_FUNC(tail_call
, struct __sk_buff
*skb
, void *map
,
164 static uint32_t BPF_FUNC(get_smp_processor_id
);
165 static uint32_t BPF_FUNC(get_numa_node_id
);
167 /* Packet misc meta data */
168 static uint32_t BPF_FUNC(get_cgroup_classid
, struct __sk_buff
*skb
);
169 static int BPF_FUNC(skb_under_cgroup
, void *map
, uint32_t index
);
171 static uint32_t BPF_FUNC(get_route_realm
, struct __sk_buff
*skb
);
172 static uint32_t BPF_FUNC(get_hash_recalc
, struct __sk_buff
*skb
);
173 static uint32_t BPF_FUNC(set_hash_invalid
, struct __sk_buff
*skb
);
175 /* Packet redirection */
176 static int BPF_FUNC(redirect
, int ifindex
, uint32_t flags
);
177 static int BPF_FUNC(clone_redirect
, struct __sk_buff
*skb
, int ifindex
,
180 /* Packet manipulation */
181 static int BPF_FUNC(skb_load_bytes
, struct __sk_buff
*skb
, uint32_t off
,
182 void *to
, uint32_t len
);
183 static int BPF_FUNC(skb_store_bytes
, struct __sk_buff
*skb
, uint32_t off
,
184 const void *from
, uint32_t len
, uint32_t flags
);
186 static int BPF_FUNC(l3_csum_replace
, struct __sk_buff
*skb
, uint32_t off
,
187 uint32_t from
, uint32_t to
, uint32_t flags
);
188 static int BPF_FUNC(l4_csum_replace
, struct __sk_buff
*skb
, uint32_t off
,
189 uint32_t from
, uint32_t to
, uint32_t flags
);
190 static int BPF_FUNC(csum_diff
, const void *from
, uint32_t from_size
,
191 const void *to
, uint32_t to_size
, uint32_t seed
);
192 static int BPF_FUNC(csum_update
, struct __sk_buff
*skb
, uint32_t wsum
);
194 static int BPF_FUNC(skb_change_type
, struct __sk_buff
*skb
, uint32_t type
);
195 static int BPF_FUNC(skb_change_proto
, struct __sk_buff
*skb
, uint32_t proto
,
197 static int BPF_FUNC(skb_change_tail
, struct __sk_buff
*skb
, uint32_t nlen
,
200 static int BPF_FUNC(skb_pull_data
, struct __sk_buff
*skb
, uint32_t len
);
202 /* Event notification */
203 static int __BPF_FUNC(skb_event_output
, struct __sk_buff
*skb
, void *map
,
204 uint64_t index
, const void *data
, uint32_t size
) =
205 (void *) BPF_FUNC_perf_event_output
;
207 /* Packet vlan encap/decap */
208 static int BPF_FUNC(skb_vlan_push
, struct __sk_buff
*skb
, uint16_t proto
,
210 static int BPF_FUNC(skb_vlan_pop
, struct __sk_buff
*skb
);
212 /* Packet tunnel encap/decap */
213 static int BPF_FUNC(skb_get_tunnel_key
, struct __sk_buff
*skb
,
214 struct bpf_tunnel_key
*to
, uint32_t size
, uint32_t flags
);
215 static int BPF_FUNC(skb_set_tunnel_key
, struct __sk_buff
*skb
,
216 const struct bpf_tunnel_key
*from
, uint32_t size
,
219 static int BPF_FUNC(skb_get_tunnel_opt
, struct __sk_buff
*skb
,
220 void *to
, uint32_t size
);
221 static int BPF_FUNC(skb_set_tunnel_opt
, struct __sk_buff
*skb
,
222 const void *from
, uint32_t size
);
224 /** LLVM built-ins, mem*() routines work for constant size */
227 # define lock_xadd(ptr, val) ((void) __sync_fetch_and_add(ptr, val))
231 # define memset(s, c, n) __builtin_memset((s), (c), (n))
235 # define memcpy(d, s, n) __builtin_memcpy((d), (s), (n))
239 # define memmove(d, s, n) __builtin_memmove((d), (s), (n))
242 /* FIXME: __builtin_memcmp() is not yet fully useable unless llvm bug
243 * https://llvm.org/bugs/show_bug.cgi?id=26218 gets resolved. Also
244 * this one would generate a reloc entry (non-map), otherwise.
248 # define memcmp(a, b, n) __builtin_memcmp((a), (b), (n))
252 unsigned long long load_byte(void *skb
, unsigned long long off
)
253 asm ("llvm.bpf.load.byte");
255 unsigned long long load_half(void *skb
, unsigned long long off
)
256 asm ("llvm.bpf.load.half");
258 unsigned long long load_word(void *skb
, unsigned long long off
)
259 asm ("llvm.bpf.load.word");
261 #endif /* __BPF_API__ */