1 /* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful, but
8 * WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10 * General Public License for more details.
13 /* A BPF sock_map is used to store sock objects. This is primarly used
14 * for doing socket redirect with BPF helper routines.
16 * A sock map may have BPF programs attached to it, currently a program
17 * used to parse packets and a program to provide a verdict and redirect
18 * decision on the packet are supported. Any programs attached to a sock
19 * map are inherited by sock objects when they are added to the map. If
20 * no BPF programs are attached the sock object may only be used for sock
23 * A sock object may be in multiple maps, but can only inherit a single
24 * parse or verdict program. If adding a sock object to a map would result
25 * in having multiple parsing programs the update will return an EBUSY error.
27 * For reference this program is similar to devmap used in XDP context
28 * reviewing these together may be useful. For an example please review
29 * ./samples/bpf/sockmap/.
31 #include <linux/bpf.h>
33 #include <linux/filter.h>
34 #include <linux/errno.h>
35 #include <linux/file.h>
36 #include <linux/kernel.h>
37 #include <linux/net.h>
38 #include <linux/skbuff.h>
39 #include <linux/workqueue.h>
40 #include <linux/list.h>
41 #include <net/strparser.h>
44 #define SOCK_CREATE_FLAG_MASK \
45 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
49 struct sock
**sock_map
;
50 struct bpf_prog
*bpf_parse
;
51 struct bpf_prog
*bpf_verdict
;
54 enum smap_psock_state
{
58 struct smap_psock_map_entry
{
59 struct list_head list
;
65 /* refcnt is used inside sk_callback_lock */
68 /* datapath variables */
69 struct sk_buff_head rxqueue
;
72 /* datapath error path cache across tx work invocations */
75 struct sk_buff
*save_skb
;
77 struct strparser strp
;
78 struct bpf_prog
*bpf_parse
;
79 struct bpf_prog
*bpf_verdict
;
80 struct list_head maps
;
82 /* Back reference used when sock callback trigger sockmap operations */
86 struct work_struct tx_work
;
87 struct work_struct gc_work
;
89 void (*save_data_ready
)(struct sock
*sk
);
90 void (*save_write_space
)(struct sock
*sk
);
91 void (*save_state_change
)(struct sock
*sk
);
94 static inline struct smap_psock
*smap_psock_sk(const struct sock
*sk
)
96 return rcu_dereference_sk_user_data(sk
);
99 /* compute the linear packet data range [data, data_end) for skb when
100 * sk_skb type programs are in use.
102 static inline void bpf_compute_data_end_sk_skb(struct sk_buff
*skb
)
104 TCP_SKB_CB(skb
)->bpf
.data_end
= skb
->data
+ skb_headlen(skb
);
113 static int smap_verdict_func(struct smap_psock
*psock
, struct sk_buff
*skb
)
115 struct bpf_prog
*prog
= READ_ONCE(psock
->bpf_verdict
);
122 /* We need to ensure that BPF metadata for maps is also cleared
123 * when we orphan the skb so that we don't have the possibility
124 * to reference a stale map.
126 TCP_SKB_CB(skb
)->bpf
.map
= NULL
;
127 skb
->sk
= psock
->sock
;
128 bpf_compute_data_pointers(skb
);
130 rc
= (*prog
->bpf_func
)(skb
, prog
->insnsi
);
134 /* Moving return codes from UAPI namespace into internal namespace */
135 return rc
== SK_PASS
?
136 (TCP_SKB_CB(skb
)->bpf
.map
? __SK_REDIRECT
: __SK_PASS
) :
140 static void smap_do_verdict(struct smap_psock
*psock
, struct sk_buff
*skb
)
145 rc
= smap_verdict_func(psock
, skb
);
148 sk
= do_sk_redirect_map(skb
);
150 struct smap_psock
*peer
= smap_psock_sk(sk
);
153 test_bit(SMAP_TX_RUNNING
, &peer
->state
) &&
154 !sock_flag(sk
, SOCK_DEAD
) &&
155 sock_writeable(sk
))) {
156 skb_set_owner_w(skb
, sk
);
157 skb_queue_tail(&peer
->rxqueue
, skb
);
158 schedule_work(&peer
->tx_work
);
162 /* Fall through and free skb otherwise */
169 static void smap_report_sk_error(struct smap_psock
*psock
, int err
)
171 struct sock
*sk
= psock
->sock
;
174 sk
->sk_error_report(sk
);
177 static void smap_release_sock(struct smap_psock
*psock
, struct sock
*sock
);
179 /* Called with lock_sock(sk) held */
180 static void smap_state_change(struct sock
*sk
)
182 struct smap_psock_map_entry
*e
, *tmp
;
183 struct smap_psock
*psock
;
184 struct socket_wq
*wq
;
189 /* Allowing transitions into an established syn_recv states allows
190 * for early binding sockets to a smap object before the connection
193 switch (sk
->sk_state
) {
196 case TCP_ESTABLISHED
:
206 /* Only release if the map entry is in fact the sock in
207 * question. There is a case where the operator deletes
208 * the sock from the map, but the TCP sock is closed before
209 * the psock is detached. Use cmpxchg to verify correct
212 psock
= smap_psock_sk(sk
);
213 if (unlikely(!psock
))
215 write_lock_bh(&sk
->sk_callback_lock
);
216 list_for_each_entry_safe(e
, tmp
, &psock
->maps
, list
) {
217 osk
= cmpxchg(e
->entry
, sk
, NULL
);
220 smap_release_sock(psock
, sk
);
223 write_unlock_bh(&sk
->sk_callback_lock
);
226 psock
= smap_psock_sk(sk
);
227 if (unlikely(!psock
))
229 smap_report_sk_error(psock
, EPIPE
);
233 wq
= rcu_dereference(sk
->sk_wq
);
234 if (skwq_has_sleeper(wq
))
235 wake_up_interruptible_all(&wq
->wait
);
239 static void smap_read_sock_strparser(struct strparser
*strp
,
242 struct smap_psock
*psock
;
245 psock
= container_of(strp
, struct smap_psock
, strp
);
246 smap_do_verdict(psock
, skb
);
250 /* Called with lock held on socket */
251 static void smap_data_ready(struct sock
*sk
)
253 struct smap_psock
*psock
;
256 psock
= smap_psock_sk(sk
);
258 write_lock_bh(&sk
->sk_callback_lock
);
259 strp_data_ready(&psock
->strp
);
260 write_unlock_bh(&sk
->sk_callback_lock
);
265 static void smap_tx_work(struct work_struct
*w
)
267 struct smap_psock
*psock
;
271 psock
= container_of(w
, struct smap_psock
, tx_work
);
273 /* lock sock to avoid losing sk_socket at some point during loop */
274 lock_sock(psock
->sock
);
275 if (psock
->save_skb
) {
276 skb
= psock
->save_skb
;
277 rem
= psock
->save_rem
;
278 off
= psock
->save_off
;
279 psock
->save_skb
= NULL
;
283 while ((skb
= skb_dequeue(&psock
->rxqueue
))) {
288 if (likely(psock
->sock
->sk_socket
))
289 n
= skb_send_sock_locked(psock
->sock
,
295 /* Retry when space is available */
296 psock
->save_skb
= skb
;
297 psock
->save_rem
= rem
;
298 psock
->save_off
= off
;
301 /* Hard errors break pipe and stop xmit */
302 smap_report_sk_error(psock
, n
? -n
: EPIPE
);
303 clear_bit(SMAP_TX_RUNNING
, &psock
->state
);
313 release_sock(psock
->sock
);
316 static void smap_write_space(struct sock
*sk
)
318 struct smap_psock
*psock
;
319 void (*write_space
)(struct sock
*sk
);
322 psock
= smap_psock_sk(sk
);
323 if (likely(psock
&& test_bit(SMAP_TX_RUNNING
, &psock
->state
)))
324 schedule_work(&psock
->tx_work
);
325 write_space
= psock
->save_write_space
;
330 static void smap_stop_sock(struct smap_psock
*psock
, struct sock
*sk
)
332 if (!psock
->strp_enabled
)
334 sk
->sk_data_ready
= psock
->save_data_ready
;
335 sk
->sk_write_space
= psock
->save_write_space
;
336 sk
->sk_state_change
= psock
->save_state_change
;
337 psock
->save_data_ready
= NULL
;
338 psock
->save_write_space
= NULL
;
339 psock
->save_state_change
= NULL
;
340 strp_stop(&psock
->strp
);
341 psock
->strp_enabled
= false;
344 static void smap_destroy_psock(struct rcu_head
*rcu
)
346 struct smap_psock
*psock
= container_of(rcu
,
347 struct smap_psock
, rcu
);
349 /* Now that a grace period has passed there is no longer
350 * any reference to this sock in the sockmap so we can
351 * destroy the psock, strparser, and bpf programs. But,
352 * because we use workqueue sync operations we can not
353 * do it in rcu context
355 schedule_work(&psock
->gc_work
);
358 static void smap_release_sock(struct smap_psock
*psock
, struct sock
*sock
)
364 smap_stop_sock(psock
, sock
);
365 clear_bit(SMAP_TX_RUNNING
, &psock
->state
);
366 rcu_assign_sk_user_data(sock
, NULL
);
367 call_rcu_sched(&psock
->rcu
, smap_destroy_psock
);
370 static int smap_parse_func_strparser(struct strparser
*strp
,
373 struct smap_psock
*psock
;
374 struct bpf_prog
*prog
;
378 psock
= container_of(strp
, struct smap_psock
, strp
);
379 prog
= READ_ONCE(psock
->bpf_parse
);
381 if (unlikely(!prog
)) {
386 /* Attach socket for bpf program to use if needed we can do this
387 * because strparser clones the skb before handing it to a upper
388 * layer, meaning skb_orphan has been called. We NULL sk on the
389 * way out to ensure we don't trigger a BUG_ON in skb/sk operations
390 * later and because we are not charging the memory of this skb to
393 skb
->sk
= psock
->sock
;
394 bpf_compute_data_pointers(skb
);
395 rc
= (*prog
->bpf_func
)(skb
, prog
->insnsi
);
402 static int smap_read_sock_done(struct strparser
*strp
, int err
)
407 static int smap_init_sock(struct smap_psock
*psock
,
410 static const struct strp_callbacks cb
= {
411 .rcv_msg
= smap_read_sock_strparser
,
412 .parse_msg
= smap_parse_func_strparser
,
413 .read_sock_done
= smap_read_sock_done
,
416 return strp_init(&psock
->strp
, sk
, &cb
);
419 static void smap_init_progs(struct smap_psock
*psock
,
420 struct bpf_stab
*stab
,
421 struct bpf_prog
*verdict
,
422 struct bpf_prog
*parse
)
424 struct bpf_prog
*orig_parse
, *orig_verdict
;
426 orig_parse
= xchg(&psock
->bpf_parse
, parse
);
427 orig_verdict
= xchg(&psock
->bpf_verdict
, verdict
);
430 bpf_prog_put(orig_verdict
);
432 bpf_prog_put(orig_parse
);
435 static void smap_start_sock(struct smap_psock
*psock
, struct sock
*sk
)
437 if (sk
->sk_data_ready
== smap_data_ready
)
439 psock
->save_data_ready
= sk
->sk_data_ready
;
440 psock
->save_write_space
= sk
->sk_write_space
;
441 psock
->save_state_change
= sk
->sk_state_change
;
442 sk
->sk_data_ready
= smap_data_ready
;
443 sk
->sk_write_space
= smap_write_space
;
444 sk
->sk_state_change
= smap_state_change
;
445 psock
->strp_enabled
= true;
448 static void sock_map_remove_complete(struct bpf_stab
*stab
)
450 bpf_map_area_free(stab
->sock_map
);
454 static void smap_gc_work(struct work_struct
*w
)
456 struct smap_psock_map_entry
*e
, *tmp
;
457 struct smap_psock
*psock
;
459 psock
= container_of(w
, struct smap_psock
, gc_work
);
461 /* no callback lock needed because we already detached sockmap ops */
462 if (psock
->strp_enabled
)
463 strp_done(&psock
->strp
);
465 cancel_work_sync(&psock
->tx_work
);
466 __skb_queue_purge(&psock
->rxqueue
);
468 /* At this point all strparser and xmit work must be complete */
469 if (psock
->bpf_parse
)
470 bpf_prog_put(psock
->bpf_parse
);
471 if (psock
->bpf_verdict
)
472 bpf_prog_put(psock
->bpf_verdict
);
474 list_for_each_entry_safe(e
, tmp
, &psock
->maps
, list
) {
479 sock_put(psock
->sock
);
483 static struct smap_psock
*smap_init_psock(struct sock
*sock
,
484 struct bpf_stab
*stab
)
486 struct smap_psock
*psock
;
488 psock
= kzalloc_node(sizeof(struct smap_psock
),
489 GFP_ATOMIC
| __GFP_NOWARN
,
490 stab
->map
.numa_node
);
492 return ERR_PTR(-ENOMEM
);
495 skb_queue_head_init(&psock
->rxqueue
);
496 INIT_WORK(&psock
->tx_work
, smap_tx_work
);
497 INIT_WORK(&psock
->gc_work
, smap_gc_work
);
498 INIT_LIST_HEAD(&psock
->maps
);
501 rcu_assign_sk_user_data(sock
, psock
);
506 static struct bpf_map
*sock_map_alloc(union bpf_attr
*attr
)
508 struct bpf_stab
*stab
;
512 if (!capable(CAP_NET_ADMIN
))
513 return ERR_PTR(-EPERM
);
515 /* check sanity of attributes */
516 if (attr
->max_entries
== 0 || attr
->key_size
!= 4 ||
517 attr
->value_size
!= 4 || attr
->map_flags
& ~SOCK_CREATE_FLAG_MASK
)
518 return ERR_PTR(-EINVAL
);
520 if (attr
->value_size
> KMALLOC_MAX_SIZE
)
521 return ERR_PTR(-E2BIG
);
523 stab
= kzalloc(sizeof(*stab
), GFP_USER
);
525 return ERR_PTR(-ENOMEM
);
527 /* mandatory map attributes */
528 stab
->map
.map_type
= attr
->map_type
;
529 stab
->map
.key_size
= attr
->key_size
;
530 stab
->map
.value_size
= attr
->value_size
;
531 stab
->map
.max_entries
= attr
->max_entries
;
532 stab
->map
.map_flags
= attr
->map_flags
;
533 stab
->map
.numa_node
= bpf_map_attr_numa_node(attr
);
535 /* make sure page count doesn't overflow */
536 cost
= (u64
) stab
->map
.max_entries
* sizeof(struct sock
*);
537 if (cost
>= U32_MAX
- PAGE_SIZE
)
540 stab
->map
.pages
= round_up(cost
, PAGE_SIZE
) >> PAGE_SHIFT
;
542 /* if map size is larger than memlock limit, reject it early */
543 err
= bpf_map_precharge_memlock(stab
->map
.pages
);
548 stab
->sock_map
= bpf_map_area_alloc(stab
->map
.max_entries
*
549 sizeof(struct sock
*),
550 stab
->map
.numa_node
);
560 static void smap_list_remove(struct smap_psock
*psock
, struct sock
**entry
)
562 struct smap_psock_map_entry
*e
, *tmp
;
564 list_for_each_entry_safe(e
, tmp
, &psock
->maps
, list
) {
565 if (e
->entry
== entry
) {
572 static void sock_map_free(struct bpf_map
*map
)
574 struct bpf_stab
*stab
= container_of(map
, struct bpf_stab
, map
);
579 /* At this point no update, lookup or delete operations can happen.
580 * However, be aware we can still get a socket state event updates,
581 * and data ready callabacks that reference the psock from sk_user_data
582 * Also psock worker threads are still in-flight. So smap_release_sock
583 * will only free the psock after cancel_sync on the worker threads
584 * and a grace period expire to ensure psock is really safe to remove.
587 for (i
= 0; i
< stab
->map
.max_entries
; i
++) {
588 struct smap_psock
*psock
;
591 sock
= xchg(&stab
->sock_map
[i
], NULL
);
595 write_lock_bh(&sock
->sk_callback_lock
);
596 psock
= smap_psock_sk(sock
);
597 /* This check handles a racing sock event that can get the
598 * sk_callback_lock before this case but after xchg happens
599 * causing the refcnt to hit zero and sock user data (psock)
600 * to be null and queued for garbage collection.
603 smap_list_remove(psock
, &stab
->sock_map
[i
]);
604 smap_release_sock(psock
, sock
);
606 write_unlock_bh(&sock
->sk_callback_lock
);
610 sock_map_remove_complete(stab
);
613 static int sock_map_get_next_key(struct bpf_map
*map
, void *key
, void *next_key
)
615 struct bpf_stab
*stab
= container_of(map
, struct bpf_stab
, map
);
616 u32 i
= key
? *(u32
*)key
: U32_MAX
;
617 u32
*next
= (u32
*)next_key
;
619 if (i
>= stab
->map
.max_entries
) {
624 if (i
== stab
->map
.max_entries
- 1)
631 struct sock
*__sock_map_lookup_elem(struct bpf_map
*map
, u32 key
)
633 struct bpf_stab
*stab
= container_of(map
, struct bpf_stab
, map
);
635 if (key
>= map
->max_entries
)
638 return READ_ONCE(stab
->sock_map
[key
]);
641 static int sock_map_delete_elem(struct bpf_map
*map
, void *key
)
643 struct bpf_stab
*stab
= container_of(map
, struct bpf_stab
, map
);
644 struct smap_psock
*psock
;
648 if (k
>= map
->max_entries
)
651 sock
= xchg(&stab
->sock_map
[k
], NULL
);
655 write_lock_bh(&sock
->sk_callback_lock
);
656 psock
= smap_psock_sk(sock
);
660 if (psock
->bpf_parse
)
661 smap_stop_sock(psock
, sock
);
662 smap_list_remove(psock
, &stab
->sock_map
[k
]);
663 smap_release_sock(psock
, sock
);
665 write_unlock_bh(&sock
->sk_callback_lock
);
669 /* Locking notes: Concurrent updates, deletes, and lookups are allowed and are
670 * done inside rcu critical sections. This ensures on updates that the psock
671 * will not be released via smap_release_sock() until concurrent updates/deletes
672 * complete. All operations operate on sock_map using cmpxchg and xchg
673 * operations to ensure we do not get stale references. Any reads into the
674 * map must be done with READ_ONCE() because of this.
676 * A psock is destroyed via call_rcu and after any worker threads are cancelled
677 * and syncd so we are certain all references from the update/lookup/delete
678 * operations as well as references in the data path are no longer in use.
680 * Psocks may exist in multiple maps, but only a single set of parse/verdict
681 * programs may be inherited from the maps it belongs to. A reference count
682 * is kept with the total number of references to the psock from all maps. The
683 * psock will not be released until this reaches zero. The psock and sock
684 * user data data use the sk_callback_lock to protect critical data structures
685 * from concurrent access. This allows us to avoid two updates from modifying
686 * the user data in sock and the lock is required anyways for modifying
687 * callbacks, we simply increase its scope slightly.
690 * - psock must always be read inside RCU critical section
691 * - sk_user_data must only be modified inside sk_callback_lock and read
692 * inside RCU critical section.
693 * - psock->maps list must only be read & modified inside sk_callback_lock
694 * - sock_map must use READ_ONCE and (cmp)xchg operations
695 * - BPF verdict/parse programs must use READ_ONCE and xchg operations
697 static int sock_map_ctx_update_elem(struct bpf_sock_ops_kern
*skops
,
699 void *key
, u64 flags
)
701 struct bpf_stab
*stab
= container_of(map
, struct bpf_stab
, map
);
702 struct smap_psock_map_entry
*e
= NULL
;
703 struct bpf_prog
*verdict
, *parse
;
704 struct sock
*osock
, *sock
;
705 struct smap_psock
*psock
;
709 if (unlikely(flags
> BPF_EXIST
))
712 if (unlikely(i
>= stab
->map
.max_entries
))
715 sock
= READ_ONCE(stab
->sock_map
[i
]);
716 if (flags
== BPF_EXIST
&& !sock
)
718 else if (flags
== BPF_NOEXIST
&& sock
)
723 /* 1. If sock map has BPF programs those will be inherited by the
724 * sock being added. If the sock is already attached to BPF programs
725 * this results in an error.
727 verdict
= READ_ONCE(stab
->bpf_verdict
);
728 parse
= READ_ONCE(stab
->bpf_parse
);
730 if (parse
&& verdict
) {
731 /* bpf prog refcnt may be zero if a concurrent attach operation
732 * removes the program after the above READ_ONCE() but before
733 * we increment the refcnt. If this is the case abort with an
736 verdict
= bpf_prog_inc_not_zero(stab
->bpf_verdict
);
738 return PTR_ERR(verdict
);
740 parse
= bpf_prog_inc_not_zero(stab
->bpf_parse
);
742 bpf_prog_put(verdict
);
743 return PTR_ERR(parse
);
747 write_lock_bh(&sock
->sk_callback_lock
);
748 psock
= smap_psock_sk(sock
);
750 /* 2. Do not allow inheriting programs if psock exists and has
751 * already inherited programs. This would create confusion on
752 * which parser/verdict program is running. If no psock exists
753 * create one. Inside sk_callback_lock to ensure concurrent create
754 * doesn't update user data.
757 if (READ_ONCE(psock
->bpf_parse
) && parse
) {
763 psock
= smap_init_psock(sock
, stab
);
765 err
= PTR_ERR(psock
);
769 set_bit(SMAP_TX_RUNNING
, &psock
->state
);
772 e
= kzalloc(sizeof(*e
), GFP_ATOMIC
| __GFP_NOWARN
);
777 e
->entry
= &stab
->sock_map
[i
];
779 /* 3. At this point we have a reference to a valid psock that is
780 * running. Attach any BPF programs needed.
782 if (parse
&& verdict
&& !psock
->strp_enabled
) {
783 err
= smap_init_sock(psock
, sock
);
786 smap_init_progs(psock
, stab
, verdict
, parse
);
787 smap_start_sock(psock
, sock
);
790 /* 4. Place psock in sockmap for use and stop any programs on
791 * the old sock assuming its not the same sock we are replacing
792 * it with. Because we can only have a single set of programs if
793 * old_sock has a strp we can stop it.
795 list_add_tail(&e
->list
, &psock
->maps
);
796 write_unlock_bh(&sock
->sk_callback_lock
);
798 osock
= xchg(&stab
->sock_map
[i
], sock
);
800 struct smap_psock
*opsock
= smap_psock_sk(osock
);
802 write_lock_bh(&osock
->sk_callback_lock
);
803 if (osock
!= sock
&& parse
)
804 smap_stop_sock(opsock
, osock
);
805 smap_list_remove(opsock
, &stab
->sock_map
[i
]);
806 smap_release_sock(opsock
, osock
);
807 write_unlock_bh(&osock
->sk_callback_lock
);
811 smap_release_sock(psock
, sock
);
814 bpf_prog_put(verdict
);
817 write_unlock_bh(&sock
->sk_callback_lock
);
822 int sock_map_prog(struct bpf_map
*map
, struct bpf_prog
*prog
, u32 type
)
824 struct bpf_stab
*stab
= container_of(map
, struct bpf_stab
, map
);
825 struct bpf_prog
*orig
;
827 if (unlikely(map
->map_type
!= BPF_MAP_TYPE_SOCKMAP
))
831 case BPF_SK_SKB_STREAM_PARSER
:
832 orig
= xchg(&stab
->bpf_parse
, prog
);
834 case BPF_SK_SKB_STREAM_VERDICT
:
835 orig
= xchg(&stab
->bpf_verdict
, prog
);
847 static void *sock_map_lookup(struct bpf_map
*map
, void *key
)
852 static int sock_map_update_elem(struct bpf_map
*map
,
853 void *key
, void *value
, u64 flags
)
855 struct bpf_sock_ops_kern skops
;
856 u32 fd
= *(u32
*)value
;
857 struct socket
*socket
;
860 socket
= sockfd_lookup(fd
, &err
);
864 skops
.sk
= socket
->sk
;
870 if (skops
.sk
->sk_type
!= SOCK_STREAM
||
871 skops
.sk
->sk_protocol
!= IPPROTO_TCP
) {
876 err
= sock_map_ctx_update_elem(&skops
, map
, key
, flags
);
881 static void sock_map_release(struct bpf_map
*map
)
883 struct bpf_stab
*stab
= container_of(map
, struct bpf_stab
, map
);
884 struct bpf_prog
*orig
;
886 orig
= xchg(&stab
->bpf_parse
, NULL
);
889 orig
= xchg(&stab
->bpf_verdict
, NULL
);
894 const struct bpf_map_ops sock_map_ops
= {
895 .map_alloc
= sock_map_alloc
,
896 .map_free
= sock_map_free
,
897 .map_lookup_elem
= sock_map_lookup
,
898 .map_get_next_key
= sock_map_get_next_key
,
899 .map_update_elem
= sock_map_update_elem
,
900 .map_delete_elem
= sock_map_delete_elem
,
901 .map_release_uref
= sock_map_release
,
904 BPF_CALL_4(bpf_sock_map_update
, struct bpf_sock_ops_kern
*, bpf_sock
,
905 struct bpf_map
*, map
, void *, key
, u64
, flags
)
907 WARN_ON_ONCE(!rcu_read_lock_held());
908 return sock_map_ctx_update_elem(bpf_sock
, map
, key
, flags
);
911 const struct bpf_func_proto bpf_sock_map_update_proto
= {
912 .func
= bpf_sock_map_update
,
915 .ret_type
= RET_INTEGER
,
916 .arg1_type
= ARG_PTR_TO_CTX
,
917 .arg2_type
= ARG_CONST_MAP_PTR
,
918 .arg3_type
= ARG_PTR_TO_MAP_KEY
,
919 .arg4_type
= ARG_ANYTHING
,