1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
8 #include <linux/filter.h>
9 #include <linux/scatterlist.h>
10 #include <linux/skbuff.h>
14 #include <net/strparser.h>
16 #define MAX_MSG_FRAGS MAX_SKB_FRAGS
17 #define NR_MSG_FRAG_IDS (MAX_MSG_FRAGS + 1)
33 /* The extra two elements:
34 * 1) used for chaining the front and sections when the list becomes
35 * partitioned (e.g. end < start). The crypto APIs require the
37 * 2) to chain tailer SG entries after the message.
39 struct scatterlist data
[MAX_MSG_FRAGS
+ 2];
41 static_assert(BITS_PER_LONG
>= NR_MSG_FRAG_IDS
);
43 /* UAPI in filter.c depends on struct sk_msg_sg being first element. */
52 struct sock
*sk_redir
;
54 struct list_head list
;
57 struct sk_psock_progs
{
58 struct bpf_prog
*msg_parser
;
59 struct bpf_prog
*stream_parser
;
60 struct bpf_prog
*stream_verdict
;
61 struct bpf_prog
*skb_verdict
;
64 enum sk_psock_state_bits
{
68 struct sk_psock_link
{
69 struct list_head list
;
74 struct sk_psock_work_state
{
82 struct sock
*sk_redir
;
87 struct sk_psock_progs progs
;
88 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
89 struct strparser strp
;
91 struct sk_buff_head ingress_skb
;
92 struct list_head ingress_msg
;
93 spinlock_t ingress_lock
;
95 struct list_head link
;
98 void (*saved_unhash
)(struct sock
*sk
);
99 void (*saved_close
)(struct sock
*sk
, long timeout
);
100 void (*saved_write_space
)(struct sock
*sk
);
101 void (*saved_data_ready
)(struct sock
*sk
);
102 int (*psock_update_sk_prot
)(struct sock
*sk
, struct sk_psock
*psock
,
104 struct proto
*sk_proto
;
105 struct mutex work_mutex
;
106 struct sk_psock_work_state work_state
;
107 struct work_struct work
;
108 struct rcu_work rwork
;
111 int sk_msg_alloc(struct sock
*sk
, struct sk_msg
*msg
, int len
,
112 int elem_first_coalesce
);
113 int sk_msg_clone(struct sock
*sk
, struct sk_msg
*dst
, struct sk_msg
*src
,
115 void sk_msg_trim(struct sock
*sk
, struct sk_msg
*msg
, int len
);
116 int sk_msg_free(struct sock
*sk
, struct sk_msg
*msg
);
117 int sk_msg_free_nocharge(struct sock
*sk
, struct sk_msg
*msg
);
118 void sk_msg_free_partial(struct sock
*sk
, struct sk_msg
*msg
, u32 bytes
);
119 void sk_msg_free_partial_nocharge(struct sock
*sk
, struct sk_msg
*msg
,
122 void sk_msg_return(struct sock
*sk
, struct sk_msg
*msg
, int bytes
);
123 void sk_msg_return_zero(struct sock
*sk
, struct sk_msg
*msg
, int bytes
);
125 int sk_msg_zerocopy_from_iter(struct sock
*sk
, struct iov_iter
*from
,
126 struct sk_msg
*msg
, u32 bytes
);
127 int sk_msg_memcopy_from_iter(struct sock
*sk
, struct iov_iter
*from
,
128 struct sk_msg
*msg
, u32 bytes
);
129 int sk_msg_recvmsg(struct sock
*sk
, struct sk_psock
*psock
, struct msghdr
*msg
,
132 static inline void sk_msg_check_to_free(struct sk_msg
*msg
, u32 i
, u32 bytes
)
134 WARN_ON(i
== msg
->sg
.end
&& bytes
);
137 static inline void sk_msg_apply_bytes(struct sk_psock
*psock
, u32 bytes
)
139 if (psock
->apply_bytes
) {
140 if (psock
->apply_bytes
< bytes
)
141 psock
->apply_bytes
= 0;
143 psock
->apply_bytes
-= bytes
;
147 static inline u32
sk_msg_iter_dist(u32 start
, u32 end
)
149 return end
>= start
? end
- start
: end
+ (NR_MSG_FRAG_IDS
- start
);
152 #define sk_msg_iter_var_prev(var) \
155 var = NR_MSG_FRAG_IDS - 1; \
160 #define sk_msg_iter_var_next(var) \
163 if (var == NR_MSG_FRAG_IDS) \
167 #define sk_msg_iter_prev(msg, which) \
168 sk_msg_iter_var_prev(msg->sg.which)
170 #define sk_msg_iter_next(msg, which) \
171 sk_msg_iter_var_next(msg->sg.which)
173 static inline void sk_msg_clear_meta(struct sk_msg
*msg
)
175 memset(&msg
->sg
, 0, offsetofend(struct sk_msg_sg
, copy
));
178 static inline void sk_msg_init(struct sk_msg
*msg
)
180 BUILD_BUG_ON(ARRAY_SIZE(msg
->sg
.data
) - 1 != NR_MSG_FRAG_IDS
);
181 memset(msg
, 0, sizeof(*msg
));
182 sg_init_marker(msg
->sg
.data
, NR_MSG_FRAG_IDS
);
185 static inline void sk_msg_xfer(struct sk_msg
*dst
, struct sk_msg
*src
,
188 dst
->sg
.data
[which
] = src
->sg
.data
[which
];
189 dst
->sg
.data
[which
].length
= size
;
190 dst
->sg
.size
+= size
;
191 src
->sg
.size
-= size
;
192 src
->sg
.data
[which
].length
-= size
;
193 src
->sg
.data
[which
].offset
+= size
;
196 static inline void sk_msg_xfer_full(struct sk_msg
*dst
, struct sk_msg
*src
)
198 memcpy(dst
, src
, sizeof(*src
));
202 static inline bool sk_msg_full(const struct sk_msg
*msg
)
204 return sk_msg_iter_dist(msg
->sg
.start
, msg
->sg
.end
) == MAX_MSG_FRAGS
;
207 static inline u32
sk_msg_elem_used(const struct sk_msg
*msg
)
209 return sk_msg_iter_dist(msg
->sg
.start
, msg
->sg
.end
);
212 static inline struct scatterlist
*sk_msg_elem(struct sk_msg
*msg
, int which
)
214 return &msg
->sg
.data
[which
];
217 static inline struct scatterlist
sk_msg_elem_cpy(struct sk_msg
*msg
, int which
)
219 return msg
->sg
.data
[which
];
222 static inline struct page
*sk_msg_page(struct sk_msg
*msg
, int which
)
224 return sg_page(sk_msg_elem(msg
, which
));
227 static inline bool sk_msg_to_ingress(const struct sk_msg
*msg
)
229 return msg
->flags
& BPF_F_INGRESS
;
232 static inline void sk_msg_compute_data_pointers(struct sk_msg
*msg
)
234 struct scatterlist
*sge
= sk_msg_elem(msg
, msg
->sg
.start
);
236 if (test_bit(msg
->sg
.start
, &msg
->sg
.copy
)) {
238 msg
->data_end
= NULL
;
240 msg
->data
= sg_virt(sge
);
241 msg
->data_end
= msg
->data
+ sge
->length
;
245 static inline void sk_msg_page_add(struct sk_msg
*msg
, struct page
*page
,
248 struct scatterlist
*sge
;
251 sge
= sk_msg_elem(msg
, msg
->sg
.end
);
252 sg_set_page(sge
, page
, len
, offset
);
255 __set_bit(msg
->sg
.end
, &msg
->sg
.copy
);
257 sk_msg_iter_next(msg
, end
);
260 static inline void sk_msg_sg_copy(struct sk_msg
*msg
, u32 i
, bool copy_state
)
264 __set_bit(i
, &msg
->sg
.copy
);
266 __clear_bit(i
, &msg
->sg
.copy
);
267 sk_msg_iter_var_next(i
);
268 if (i
== msg
->sg
.end
)
273 static inline void sk_msg_sg_copy_set(struct sk_msg
*msg
, u32 start
)
275 sk_msg_sg_copy(msg
, start
, true);
278 static inline void sk_msg_sg_copy_clear(struct sk_msg
*msg
, u32 start
)
280 sk_msg_sg_copy(msg
, start
, false);
283 static inline struct sk_psock
*sk_psock(const struct sock
*sk
)
285 return rcu_dereference_sk_user_data(sk
);
288 static inline void sk_psock_set_state(struct sk_psock
*psock
,
289 enum sk_psock_state_bits bit
)
291 set_bit(bit
, &psock
->state
);
294 static inline void sk_psock_clear_state(struct sk_psock
*psock
,
295 enum sk_psock_state_bits bit
)
297 clear_bit(bit
, &psock
->state
);
300 static inline bool sk_psock_test_state(const struct sk_psock
*psock
,
301 enum sk_psock_state_bits bit
)
303 return test_bit(bit
, &psock
->state
);
306 static inline void sock_drop(struct sock
*sk
, struct sk_buff
*skb
)
308 sk_drops_add(sk
, skb
);
312 static inline void drop_sk_msg(struct sk_psock
*psock
, struct sk_msg
*msg
)
315 sock_drop(psock
->sk
, msg
->skb
);
319 static inline void sk_psock_queue_msg(struct sk_psock
*psock
,
322 spin_lock_bh(&psock
->ingress_lock
);
323 if (sk_psock_test_state(psock
, SK_PSOCK_TX_ENABLED
))
324 list_add_tail(&msg
->list
, &psock
->ingress_msg
);
326 drop_sk_msg(psock
, msg
);
327 spin_unlock_bh(&psock
->ingress_lock
);
330 static inline struct sk_msg
*sk_psock_dequeue_msg(struct sk_psock
*psock
)
334 spin_lock_bh(&psock
->ingress_lock
);
335 msg
= list_first_entry_or_null(&psock
->ingress_msg
, struct sk_msg
, list
);
337 list_del(&msg
->list
);
338 spin_unlock_bh(&psock
->ingress_lock
);
342 static inline struct sk_msg
*sk_psock_peek_msg(struct sk_psock
*psock
)
346 spin_lock_bh(&psock
->ingress_lock
);
347 msg
= list_first_entry_or_null(&psock
->ingress_msg
, struct sk_msg
, list
);
348 spin_unlock_bh(&psock
->ingress_lock
);
352 static inline struct sk_msg
*sk_psock_next_msg(struct sk_psock
*psock
,
357 spin_lock_bh(&psock
->ingress_lock
);
358 if (list_is_last(&msg
->list
, &psock
->ingress_msg
))
361 ret
= list_next_entry(msg
, list
);
362 spin_unlock_bh(&psock
->ingress_lock
);
366 static inline bool sk_psock_queue_empty(const struct sk_psock
*psock
)
368 return psock
? list_empty(&psock
->ingress_msg
) : true;
371 static inline void kfree_sk_msg(struct sk_msg
*msg
)
374 consume_skb(msg
->skb
);
378 static inline void sk_psock_report_error(struct sk_psock
*psock
, int err
)
380 struct sock
*sk
= psock
->sk
;
386 struct sk_psock
*sk_psock_init(struct sock
*sk
, int node
);
387 void sk_psock_stop(struct sk_psock
*psock
, bool wait
);
389 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
390 int sk_psock_init_strp(struct sock
*sk
, struct sk_psock
*psock
);
391 void sk_psock_start_strp(struct sock
*sk
, struct sk_psock
*psock
);
392 void sk_psock_stop_strp(struct sock
*sk
, struct sk_psock
*psock
);
394 static inline int sk_psock_init_strp(struct sock
*sk
, struct sk_psock
*psock
)
399 static inline void sk_psock_start_strp(struct sock
*sk
, struct sk_psock
*psock
)
403 static inline void sk_psock_stop_strp(struct sock
*sk
, struct sk_psock
*psock
)
408 void sk_psock_start_verdict(struct sock
*sk
, struct sk_psock
*psock
);
409 void sk_psock_stop_verdict(struct sock
*sk
, struct sk_psock
*psock
);
411 int sk_psock_msg_verdict(struct sock
*sk
, struct sk_psock
*psock
,
414 static inline struct sk_psock_link
*sk_psock_init_link(void)
416 return kzalloc(sizeof(struct sk_psock_link
),
417 GFP_ATOMIC
| __GFP_NOWARN
);
420 static inline void sk_psock_free_link(struct sk_psock_link
*link
)
425 struct sk_psock_link
*sk_psock_link_pop(struct sk_psock
*psock
);
427 static inline void sk_psock_cork_free(struct sk_psock
*psock
)
430 sk_msg_free(psock
->sk
, psock
->cork
);
436 static inline void sk_psock_restore_proto(struct sock
*sk
,
437 struct sk_psock
*psock
)
439 if (psock
->psock_update_sk_prot
)
440 psock
->psock_update_sk_prot(sk
, psock
, true);
443 static inline struct sk_psock
*sk_psock_get(struct sock
*sk
)
445 struct sk_psock
*psock
;
448 psock
= sk_psock(sk
);
449 if (psock
&& !refcount_inc_not_zero(&psock
->refcnt
))
455 void sk_psock_drop(struct sock
*sk
, struct sk_psock
*psock
);
457 static inline void sk_psock_put(struct sock
*sk
, struct sk_psock
*psock
)
459 if (refcount_dec_and_test(&psock
->refcnt
))
460 sk_psock_drop(sk
, psock
);
463 static inline void sk_psock_data_ready(struct sock
*sk
, struct sk_psock
*psock
)
465 if (psock
->saved_data_ready
)
466 psock
->saved_data_ready(sk
);
468 sk
->sk_data_ready(sk
);
471 static inline void psock_set_prog(struct bpf_prog
**pprog
,
472 struct bpf_prog
*prog
)
474 prog
= xchg(pprog
, prog
);
479 static inline int psock_replace_prog(struct bpf_prog
**pprog
,
480 struct bpf_prog
*prog
,
481 struct bpf_prog
*old
)
483 if (cmpxchg(pprog
, old
, prog
) != old
)
492 static inline void psock_progs_drop(struct sk_psock_progs
*progs
)
494 psock_set_prog(&progs
->msg_parser
, NULL
);
495 psock_set_prog(&progs
->stream_parser
, NULL
);
496 psock_set_prog(&progs
->stream_verdict
, NULL
);
497 psock_set_prog(&progs
->skb_verdict
, NULL
);
500 int sk_psock_tls_strp_read(struct sk_psock
*psock
, struct sk_buff
*skb
);
502 static inline bool sk_psock_strp_enabled(struct sk_psock
*psock
)
506 return !!psock
->saved_data_ready
;
509 #if IS_ENABLED(CONFIG_NET_SOCK_MSG)
511 /* We only have one bit so far. */
512 #define BPF_F_PTR_MASK ~(BPF_F_INGRESS)
514 static inline bool skb_bpf_ingress(const struct sk_buff
*skb
)
516 unsigned long sk_redir
= skb
->_sk_redir
;
518 return sk_redir
& BPF_F_INGRESS
;
521 static inline void skb_bpf_set_ingress(struct sk_buff
*skb
)
523 skb
->_sk_redir
|= BPF_F_INGRESS
;
526 static inline void skb_bpf_set_redir(struct sk_buff
*skb
, struct sock
*sk_redir
,
529 skb
->_sk_redir
= (unsigned long)sk_redir
;
531 skb
->_sk_redir
|= BPF_F_INGRESS
;
534 static inline struct sock
*skb_bpf_redirect_fetch(const struct sk_buff
*skb
)
536 unsigned long sk_redir
= skb
->_sk_redir
;
538 return (struct sock
*)(sk_redir
& BPF_F_PTR_MASK
);
541 static inline void skb_bpf_redirect_clear(struct sk_buff
*skb
)
545 #endif /* CONFIG_NET_SOCK_MSG */
546 #endif /* _LINUX_SKMSG_H */