]>
Commit | Line | Data |
---|---|---|
de9cbbaa RG |
1 | //SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/bpf-cgroup.h> | |
3 | #include <linux/bpf.h> | |
4 | #include <linux/bug.h> | |
5 | #include <linux/filter.h> | |
6 | #include <linux/mm.h> | |
7 | #include <linux/rbtree.h> | |
8 | #include <linux/slab.h> | |
9 | ||
f294b37e RG |
10 | DEFINE_PER_CPU(struct bpf_cgroup_storage*, |
11 | bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); | |
aa0ad5b0 | 12 | |
de9cbbaa RG |
13 | #ifdef CONFIG_CGROUP_BPF |
14 | ||
15 | #define LOCAL_STORAGE_CREATE_FLAG_MASK \ | |
16 | (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) | |
17 | ||
18 | struct bpf_cgroup_storage_map { | |
19 | struct bpf_map map; | |
20 | ||
21 | spinlock_t lock; | |
22 | struct bpf_prog *prog; | |
23 | struct rb_root root; | |
24 | struct list_head list; | |
25 | }; | |
26 | ||
27 | static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map) | |
28 | { | |
29 | return container_of(map, struct bpf_cgroup_storage_map, map); | |
30 | } | |
31 | ||
32 | static int bpf_cgroup_storage_key_cmp( | |
33 | const struct bpf_cgroup_storage_key *key1, | |
34 | const struct bpf_cgroup_storage_key *key2) | |
35 | { | |
36 | if (key1->cgroup_inode_id < key2->cgroup_inode_id) | |
37 | return -1; | |
38 | else if (key1->cgroup_inode_id > key2->cgroup_inode_id) | |
39 | return 1; | |
40 | else if (key1->attach_type < key2->attach_type) | |
41 | return -1; | |
42 | else if (key1->attach_type > key2->attach_type) | |
43 | return 1; | |
44 | return 0; | |
45 | } | |
46 | ||
47 | static struct bpf_cgroup_storage *cgroup_storage_lookup( | |
48 | struct bpf_cgroup_storage_map *map, struct bpf_cgroup_storage_key *key, | |
49 | bool locked) | |
50 | { | |
51 | struct rb_root *root = &map->root; | |
52 | struct rb_node *node; | |
53 | ||
54 | if (!locked) | |
55 | spin_lock_bh(&map->lock); | |
56 | ||
57 | node = root->rb_node; | |
58 | while (node) { | |
59 | struct bpf_cgroup_storage *storage; | |
60 | ||
61 | storage = container_of(node, struct bpf_cgroup_storage, node); | |
62 | ||
63 | switch (bpf_cgroup_storage_key_cmp(key, &storage->key)) { | |
64 | case -1: | |
65 | node = node->rb_left; | |
66 | break; | |
67 | case 1: | |
68 | node = node->rb_right; | |
69 | break; | |
70 | default: | |
71 | if (!locked) | |
72 | spin_unlock_bh(&map->lock); | |
73 | return storage; | |
74 | } | |
75 | } | |
76 | ||
77 | if (!locked) | |
78 | spin_unlock_bh(&map->lock); | |
79 | ||
80 | return NULL; | |
81 | } | |
82 | ||
83 | static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map, | |
84 | struct bpf_cgroup_storage *storage) | |
85 | { | |
86 | struct rb_root *root = &map->root; | |
87 | struct rb_node **new = &(root->rb_node), *parent = NULL; | |
88 | ||
89 | while (*new) { | |
90 | struct bpf_cgroup_storage *this; | |
91 | ||
92 | this = container_of(*new, struct bpf_cgroup_storage, node); | |
93 | ||
94 | parent = *new; | |
95 | switch (bpf_cgroup_storage_key_cmp(&storage->key, &this->key)) { | |
96 | case -1: | |
97 | new = &((*new)->rb_left); | |
98 | break; | |
99 | case 1: | |
100 | new = &((*new)->rb_right); | |
101 | break; | |
102 | default: | |
103 | return -EEXIST; | |
104 | } | |
105 | } | |
106 | ||
107 | rb_link_node(&storage->node, parent, new); | |
108 | rb_insert_color(&storage->node, root); | |
109 | ||
110 | return 0; | |
111 | } | |
112 | ||
113 | static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *_key) | |
114 | { | |
115 | struct bpf_cgroup_storage_map *map = map_to_storage(_map); | |
116 | struct bpf_cgroup_storage_key *key = _key; | |
117 | struct bpf_cgroup_storage *storage; | |
118 | ||
119 | storage = cgroup_storage_lookup(map, key, false); | |
120 | if (!storage) | |
121 | return NULL; | |
122 | ||
123 | return &READ_ONCE(storage->buf)->data[0]; | |
124 | } | |
125 | ||
126 | static int cgroup_storage_update_elem(struct bpf_map *map, void *_key, | |
127 | void *value, u64 flags) | |
128 | { | |
129 | struct bpf_cgroup_storage_key *key = _key; | |
130 | struct bpf_cgroup_storage *storage; | |
131 | struct bpf_storage_buffer *new; | |
132 | ||
133 | if (flags & BPF_NOEXIST) | |
134 | return -EINVAL; | |
135 | ||
136 | storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, | |
137 | key, false); | |
138 | if (!storage) | |
139 | return -ENOENT; | |
140 | ||
141 | new = kmalloc_node(sizeof(struct bpf_storage_buffer) + | |
142 | map->value_size, __GFP_ZERO | GFP_USER, | |
143 | map->numa_node); | |
144 | if (!new) | |
145 | return -ENOMEM; | |
146 | ||
147 | memcpy(&new->data[0], value, map->value_size); | |
148 | ||
149 | new = xchg(&storage->buf, new); | |
150 | kfree_rcu(new, rcu); | |
151 | ||
152 | return 0; | |
153 | } | |
154 | ||
155 | static int cgroup_storage_get_next_key(struct bpf_map *_map, void *_key, | |
156 | void *_next_key) | |
157 | { | |
158 | struct bpf_cgroup_storage_map *map = map_to_storage(_map); | |
159 | struct bpf_cgroup_storage_key *key = _key; | |
160 | struct bpf_cgroup_storage_key *next = _next_key; | |
161 | struct bpf_cgroup_storage *storage; | |
162 | ||
163 | spin_lock_bh(&map->lock); | |
164 | ||
165 | if (list_empty(&map->list)) | |
166 | goto enoent; | |
167 | ||
168 | if (key) { | |
169 | storage = cgroup_storage_lookup(map, key, true); | |
170 | if (!storage) | |
171 | goto enoent; | |
172 | ||
173 | storage = list_next_entry(storage, list); | |
174 | if (!storage) | |
175 | goto enoent; | |
176 | } else { | |
177 | storage = list_first_entry(&map->list, | |
178 | struct bpf_cgroup_storage, list); | |
179 | } | |
180 | ||
181 | spin_unlock_bh(&map->lock); | |
182 | next->attach_type = storage->key.attach_type; | |
183 | next->cgroup_inode_id = storage->key.cgroup_inode_id; | |
184 | return 0; | |
185 | ||
186 | enoent: | |
187 | spin_unlock_bh(&map->lock); | |
188 | return -ENOENT; | |
189 | } | |
190 | ||
191 | static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr) | |
192 | { | |
193 | int numa_node = bpf_map_attr_numa_node(attr); | |
194 | struct bpf_cgroup_storage_map *map; | |
195 | ||
196 | if (attr->key_size != sizeof(struct bpf_cgroup_storage_key)) | |
197 | return ERR_PTR(-EINVAL); | |
198 | ||
199 | if (attr->value_size > PAGE_SIZE) | |
200 | return ERR_PTR(-E2BIG); | |
201 | ||
202 | if (attr->map_flags & ~LOCAL_STORAGE_CREATE_FLAG_MASK) | |
203 | /* reserved bits should not be used */ | |
204 | return ERR_PTR(-EINVAL); | |
205 | ||
206 | if (attr->max_entries) | |
207 | /* max_entries is not used and enforced to be 0 */ | |
208 | return ERR_PTR(-EINVAL); | |
209 | ||
210 | map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map), | |
211 | __GFP_ZERO | GFP_USER, numa_node); | |
212 | if (!map) | |
213 | return ERR_PTR(-ENOMEM); | |
214 | ||
215 | map->map.pages = round_up(sizeof(struct bpf_cgroup_storage_map), | |
216 | PAGE_SIZE) >> PAGE_SHIFT; | |
217 | ||
218 | /* copy mandatory map attributes */ | |
219 | bpf_map_init_from_attr(&map->map, attr); | |
220 | ||
221 | spin_lock_init(&map->lock); | |
222 | map->root = RB_ROOT; | |
223 | INIT_LIST_HEAD(&map->list); | |
224 | ||
225 | return &map->map; | |
226 | } | |
227 | ||
228 | static void cgroup_storage_map_free(struct bpf_map *_map) | |
229 | { | |
230 | struct bpf_cgroup_storage_map *map = map_to_storage(_map); | |
231 | ||
232 | WARN_ON(!RB_EMPTY_ROOT(&map->root)); | |
233 | WARN_ON(!list_empty(&map->list)); | |
234 | ||
235 | kfree(map); | |
236 | } | |
237 | ||
238 | static int cgroup_storage_delete_elem(struct bpf_map *map, void *key) | |
239 | { | |
240 | return -EINVAL; | |
241 | } | |
242 | ||
243 | const struct bpf_map_ops cgroup_storage_map_ops = { | |
244 | .map_alloc = cgroup_storage_map_alloc, | |
245 | .map_free = cgroup_storage_map_free, | |
246 | .map_get_next_key = cgroup_storage_get_next_key, | |
247 | .map_lookup_elem = cgroup_storage_lookup_elem, | |
248 | .map_update_elem = cgroup_storage_update_elem, | |
249 | .map_delete_elem = cgroup_storage_delete_elem, | |
e8d2bec0 | 250 | .map_check_btf = map_check_no_btf, |
de9cbbaa RG |
251 | }; |
252 | ||
253 | int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map) | |
254 | { | |
8bad74f9 | 255 | enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map); |
de9cbbaa RG |
256 | struct bpf_cgroup_storage_map *map = map_to_storage(_map); |
257 | int ret = -EBUSY; | |
258 | ||
259 | spin_lock_bh(&map->lock); | |
260 | ||
261 | if (map->prog && map->prog != prog) | |
262 | goto unlock; | |
8bad74f9 RG |
263 | if (prog->aux->cgroup_storage[stype] && |
264 | prog->aux->cgroup_storage[stype] != _map) | |
de9cbbaa RG |
265 | goto unlock; |
266 | ||
267 | map->prog = prog; | |
8bad74f9 | 268 | prog->aux->cgroup_storage[stype] = _map; |
de9cbbaa RG |
269 | ret = 0; |
270 | unlock: | |
271 | spin_unlock_bh(&map->lock); | |
272 | ||
273 | return ret; | |
274 | } | |
275 | ||
276 | void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *_map) | |
277 | { | |
8bad74f9 | 278 | enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map); |
de9cbbaa RG |
279 | struct bpf_cgroup_storage_map *map = map_to_storage(_map); |
280 | ||
281 | spin_lock_bh(&map->lock); | |
282 | if (map->prog == prog) { | |
8bad74f9 | 283 | WARN_ON(prog->aux->cgroup_storage[stype] != _map); |
de9cbbaa | 284 | map->prog = NULL; |
8bad74f9 | 285 | prog->aux->cgroup_storage[stype] = NULL; |
de9cbbaa RG |
286 | } |
287 | spin_unlock_bh(&map->lock); | |
288 | } | |
289 | ||
8bad74f9 RG |
290 | struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog, |
291 | enum bpf_cgroup_storage_type stype) | |
de9cbbaa RG |
292 | { |
293 | struct bpf_cgroup_storage *storage; | |
294 | struct bpf_map *map; | |
295 | u32 pages; | |
296 | ||
8bad74f9 | 297 | map = prog->aux->cgroup_storage[stype]; |
de9cbbaa RG |
298 | if (!map) |
299 | return NULL; | |
300 | ||
301 | pages = round_up(sizeof(struct bpf_cgroup_storage) + | |
302 | sizeof(struct bpf_storage_buffer) + | |
303 | map->value_size, PAGE_SIZE) >> PAGE_SHIFT; | |
304 | if (bpf_map_charge_memlock(map, pages)) | |
305 | return ERR_PTR(-EPERM); | |
306 | ||
307 | storage = kmalloc_node(sizeof(struct bpf_cgroup_storage), | |
308 | __GFP_ZERO | GFP_USER, map->numa_node); | |
309 | if (!storage) { | |
310 | bpf_map_uncharge_memlock(map, pages); | |
311 | return ERR_PTR(-ENOMEM); | |
312 | } | |
313 | ||
314 | storage->buf = kmalloc_node(sizeof(struct bpf_storage_buffer) + | |
315 | map->value_size, __GFP_ZERO | GFP_USER, | |
316 | map->numa_node); | |
317 | if (!storage->buf) { | |
318 | bpf_map_uncharge_memlock(map, pages); | |
319 | kfree(storage); | |
320 | return ERR_PTR(-ENOMEM); | |
321 | } | |
322 | ||
323 | storage->map = (struct bpf_cgroup_storage_map *)map; | |
324 | ||
325 | return storage; | |
326 | } | |
327 | ||
328 | void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) | |
329 | { | |
330 | u32 pages; | |
331 | struct bpf_map *map; | |
332 | ||
333 | if (!storage) | |
334 | return; | |
335 | ||
336 | map = &storage->map->map; | |
337 | pages = round_up(sizeof(struct bpf_cgroup_storage) + | |
338 | sizeof(struct bpf_storage_buffer) + | |
339 | map->value_size, PAGE_SIZE) >> PAGE_SHIFT; | |
340 | bpf_map_uncharge_memlock(map, pages); | |
341 | ||
342 | kfree_rcu(storage->buf, rcu); | |
343 | kfree_rcu(storage, rcu); | |
344 | } | |
345 | ||
346 | void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, | |
347 | struct cgroup *cgroup, | |
348 | enum bpf_attach_type type) | |
349 | { | |
350 | struct bpf_cgroup_storage_map *map; | |
351 | ||
352 | if (!storage) | |
353 | return; | |
354 | ||
355 | storage->key.attach_type = type; | |
356 | storage->key.cgroup_inode_id = cgroup->kn->id.id; | |
357 | ||
358 | map = storage->map; | |
359 | ||
360 | spin_lock_bh(&map->lock); | |
361 | WARN_ON(cgroup_storage_insert(map, storage)); | |
362 | list_add(&storage->list, &map->list); | |
363 | spin_unlock_bh(&map->lock); | |
364 | } | |
365 | ||
366 | void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) | |
367 | { | |
368 | struct bpf_cgroup_storage_map *map; | |
369 | struct rb_root *root; | |
370 | ||
371 | if (!storage) | |
372 | return; | |
373 | ||
374 | map = storage->map; | |
375 | ||
376 | spin_lock_bh(&map->lock); | |
377 | root = &map->root; | |
378 | rb_erase(&storage->node, root); | |
379 | ||
380 | list_del(&storage->list); | |
381 | spin_unlock_bh(&map->lock); | |
382 | } | |
383 | ||
384 | #endif |