6 * This file can be included into eBPF kernel programs. It contains
7 * a couple of useful helper functions, map/section ABI (bpf_elf.h),
8 * misc macros and some eBPF specific LLVM built-ins.
13 #include <linux/pkt_cls.h>
14 #include <linux/bpf.h>
15 #include <linux/filter.h>
17 #include <asm/byteorder.h>
24 # define __stringify(X) #X
27 #ifndef __maybe_unused
28 # define __maybe_unused __attribute__((__unused__))
32 # define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
36 # define likely(X) __builtin_expect(!!(X), 1)
40 # define unlikely(X) __builtin_expect(!!(X), 0)
44 # define htons(X) __constant_htons((X))
48 # define ntohs(X) __constant_ntohs((X))
52 # define htonl(X) __constant_htonl((X))
56 # define ntohl(X) __constant_ntohl((X))
60 # define __inline__ __attribute__((always_inline))
63 /** Section helper macros. */
66 # define __section(NAME) \
67 __attribute__((section(NAME), used))
70 #ifndef __section_tail
71 # define __section_tail(ID, KEY) \
72 __section(__stringify(ID) "/" __stringify(KEY))
75 #ifndef __section_xdp_entry
76 # define __section_xdp_entry \
77 __section(ELF_SECTION_PROG)
80 #ifndef __section_cls_entry
81 # define __section_cls_entry \
82 __section(ELF_SECTION_CLASSIFIER)
85 #ifndef __section_act_entry
86 # define __section_act_entry \
87 __section(ELF_SECTION_ACTION)
90 #ifndef __section_license
91 # define __section_license \
92 __section(ELF_SECTION_LICENSE)
95 #ifndef __section_maps
96 # define __section_maps \
97 __section(ELF_SECTION_MAPS)
100 /** Declaration helper macros. */
103 # define BPF_LICENSE(NAME) \
104 char ____license[] __section_license = NAME
107 /** Classifier helper */
109 #ifndef BPF_H_DEFAULT
110 # define BPF_H_DEFAULT -1
113 /** BPF helper functions for tc. Individual flags are in linux/bpf.h */
116 # define __BPF_FUNC(NAME, ...) \
117 (* NAME)(__VA_ARGS__) __maybe_unused
121 # define BPF_FUNC(NAME, ...) \
122 __BPF_FUNC(NAME, __VA_ARGS__) = (void *) BPF_FUNC_##NAME
125 /* Map access/manipulation */
126 static void *BPF_FUNC(map_lookup_elem
, void *map
, const void *key
);
127 static int BPF_FUNC(map_update_elem
, void *map
, const void *key
,
128 const void *value
, uint32_t flags
);
129 static int BPF_FUNC(map_delete_elem
, void *map
, const void *key
);
132 static uint64_t BPF_FUNC(ktime_get_ns
);
136 /* FIXME: __attribute__ ((format(printf, 1, 3))) not possible unless
137 * llvm bug https://llvm.org/bugs/show_bug.cgi?id=26243 gets resolved.
138 * It would require ____fmt to be made const, which generates a reloc
141 static void BPF_FUNC(trace_printk
, const char *fmt
, int fmt_size
, ...);
144 # define printt(fmt, ...) \
146 char ____fmt[] = fmt; \
147 trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
152 static uint32_t BPF_FUNC(get_prandom_u32
);
155 static void BPF_FUNC(tail_call
, struct __sk_buff
*skb
, void *map
,
159 static uint32_t BPF_FUNC(get_smp_processor_id
);
160 static uint32_t BPF_FUNC(get_numa_node_id
);
162 /* Packet misc meta data */
163 static uint32_t BPF_FUNC(get_cgroup_classid
, struct __sk_buff
*skb
);
164 static int BPF_FUNC(skb_under_cgroup
, void *map
, uint32_t index
);
166 static uint32_t BPF_FUNC(get_route_realm
, struct __sk_buff
*skb
);
167 static uint32_t BPF_FUNC(get_hash_recalc
, struct __sk_buff
*skb
);
168 static uint32_t BPF_FUNC(set_hash_invalid
, struct __sk_buff
*skb
);
170 /* Packet redirection */
171 static int BPF_FUNC(redirect
, int ifindex
, uint32_t flags
);
172 static int BPF_FUNC(clone_redirect
, struct __sk_buff
*skb
, int ifindex
,
175 /* Packet manipulation */
176 static int BPF_FUNC(skb_load_bytes
, struct __sk_buff
*skb
, uint32_t off
,
177 void *to
, uint32_t len
);
178 static int BPF_FUNC(skb_store_bytes
, struct __sk_buff
*skb
, uint32_t off
,
179 const void *from
, uint32_t len
, uint32_t flags
);
181 static int BPF_FUNC(l3_csum_replace
, struct __sk_buff
*skb
, uint32_t off
,
182 uint32_t from
, uint32_t to
, uint32_t flags
);
183 static int BPF_FUNC(l4_csum_replace
, struct __sk_buff
*skb
, uint32_t off
,
184 uint32_t from
, uint32_t to
, uint32_t flags
);
185 static int BPF_FUNC(csum_diff
, const void *from
, uint32_t from_size
,
186 const void *to
, uint32_t to_size
, uint32_t seed
);
187 static int BPF_FUNC(csum_update
, struct __sk_buff
*skb
, uint32_t wsum
);
189 static int BPF_FUNC(skb_change_type
, struct __sk_buff
*skb
, uint32_t type
);
190 static int BPF_FUNC(skb_change_proto
, struct __sk_buff
*skb
, uint32_t proto
,
192 static int BPF_FUNC(skb_change_tail
, struct __sk_buff
*skb
, uint32_t nlen
,
195 static int BPF_FUNC(skb_pull_data
, struct __sk_buff
*skb
, uint32_t len
);
197 /* Event notification */
198 static int __BPF_FUNC(skb_event_output
, struct __sk_buff
*skb
, void *map
,
199 uint64_t index
, const void *data
, uint32_t size
) =
200 (void *) BPF_FUNC_perf_event_output
;
202 /* Packet vlan encap/decap */
203 static int BPF_FUNC(skb_vlan_push
, struct __sk_buff
*skb
, uint16_t proto
,
205 static int BPF_FUNC(skb_vlan_pop
, struct __sk_buff
*skb
);
207 /* Packet tunnel encap/decap */
208 static int BPF_FUNC(skb_get_tunnel_key
, struct __sk_buff
*skb
,
209 struct bpf_tunnel_key
*to
, uint32_t size
, uint32_t flags
);
210 static int BPF_FUNC(skb_set_tunnel_key
, struct __sk_buff
*skb
,
211 const struct bpf_tunnel_key
*from
, uint32_t size
,
214 static int BPF_FUNC(skb_get_tunnel_opt
, struct __sk_buff
*skb
,
215 void *to
, uint32_t size
);
216 static int BPF_FUNC(skb_set_tunnel_opt
, struct __sk_buff
*skb
,
217 const void *from
, uint32_t size
);
219 /** LLVM built-ins, mem*() routines work for constant size */
222 # define lock_xadd(ptr, val) ((void) __sync_fetch_and_add(ptr, val))
226 # define memset(s, c, n) __builtin_memset((s), (c), (n))
230 # define memcpy(d, s, n) __builtin_memcpy((d), (s), (n))
234 # define memmove(d, s, n) __builtin_memmove((d), (s), (n))
237 /* FIXME: __builtin_memcmp() is not yet fully useable unless llvm bug
238 * https://llvm.org/bugs/show_bug.cgi?id=26218 gets resolved. Also
239 * this one would generate a reloc entry (non-map), otherwise.
243 # define memcmp(a, b, n) __builtin_memcmp((a), (b), (n))
247 unsigned long long load_byte(void *skb
, unsigned long long off
)
248 asm ("llvm.bpf.load.byte");
250 unsigned long long load_half(void *skb
, unsigned long long off
)
251 asm ("llvm.bpf.load.half");
253 unsigned long long load_word(void *skb
, unsigned long long off
)
254 asm ("llvm.bpf.load.word");
256 #endif /* __BPF_API__ */