4 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2
8 * as published by the Free Software Foundation.
11 #include <linux/bpf.h>
12 #include <linux/errno.h>
13 #include <linux/errqueue.h>
14 #include <linux/file.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/net.h>
19 #include <linux/netdevice.h>
20 #include <linux/poll.h>
21 #include <linux/rculist.h>
22 #include <linux/skbuff.h>
23 #include <linux/socket.h>
24 #include <linux/uaccess.h>
25 #include <linux/workqueue.h>
26 #include <net/strparser.h>
27 #include <net/netns/generic.h>
30 static struct workqueue_struct
*strp_wq
;
33 /* Internal cb structure. struct strp_msg must be first for passing
40 static inline struct _strp_msg
*_strp_msg(struct sk_buff
*skb
)
42 return (struct _strp_msg
*)((void *)skb
->cb
+
43 offsetof(struct qdisc_skb_cb
, data
));
47 static void strp_abort_strp(struct strparser
*strp
, int err
)
49 /* Unrecoverable error in receive */
51 cancel_delayed_work(&strp
->msg_timer_work
);
59 struct sock
*sk
= strp
->sk
;
61 /* Report an error on the lower socket */
63 sk
->sk_error_report(sk
);
67 static void strp_start_timer(struct strparser
*strp
, long timeo
)
69 if (timeo
&& timeo
!= LONG_MAX
)
70 mod_delayed_work(strp_wq
, &strp
->msg_timer_work
, timeo
);
74 static void strp_parser_err(struct strparser
*strp
, int err
,
75 read_descriptor_t
*desc
)
78 kfree_skb(strp
->skb_head
);
79 strp
->skb_head
= NULL
;
80 strp
->cb
.abort_parser(strp
, err
);
83 static inline int strp_peek_len(struct strparser
*strp
)
86 struct socket
*sock
= strp
->sk
->sk_socket
;
88 return sock
->ops
->peek_len(sock
);
91 /* If we don't have an associated socket there's nothing to peek.
92 * Return int max to avoid stopping the strparser.
98 /* Lower socket lock held */
99 static int __strp_recv(read_descriptor_t
*desc
, struct sk_buff
*orig_skb
,
100 unsigned int orig_offset
, size_t orig_len
,
101 size_t max_msg_size
, long timeo
)
103 struct strparser
*strp
= (struct strparser
*)desc
->arg
.data
;
104 struct _strp_msg
*stm
;
105 struct sk_buff
*head
, *skb
;
106 size_t eaten
= 0, cand_len
;
109 bool cloned_orig
= false;
114 head
= strp
->skb_head
;
116 /* Message already in progress */
117 if (unlikely(orig_offset
)) {
118 /* Getting data with a non-zero offset when a message is
119 * in progress is not expected. If it does happen, we
120 * need to clone and pull since we can't deal with
121 * offsets in the skbs for a message expect in the head.
123 orig_skb
= skb_clone(orig_skb
, GFP_ATOMIC
);
125 STRP_STATS_INCR(strp
->stats
.mem_fail
);
126 desc
->error
= -ENOMEM
;
129 if (!pskb_pull(orig_skb
, orig_offset
)) {
130 STRP_STATS_INCR(strp
->stats
.mem_fail
);
132 desc
->error
= -ENOMEM
;
139 if (!strp
->skb_nextp
) {
140 /* We are going to append to the frags_list of head.
141 * Need to unshare the frag_list.
143 err
= skb_unclone(head
, GFP_ATOMIC
);
145 STRP_STATS_INCR(strp
->stats
.mem_fail
);
150 if (unlikely(skb_shinfo(head
)->frag_list
)) {
151 /* We can't append to an sk_buff that already
152 * has a frag_list. We create a new head, point
153 * the frag_list of that to the old head, and
154 * then are able to use the old head->next for
155 * appending to the message.
157 if (WARN_ON(head
->next
)) {
158 desc
->error
= -EINVAL
;
162 skb
= alloc_skb(0, GFP_ATOMIC
);
164 STRP_STATS_INCR(strp
->stats
.mem_fail
);
165 desc
->error
= -ENOMEM
;
168 skb
->len
= head
->len
;
169 skb
->data_len
= head
->len
;
170 skb
->truesize
= head
->truesize
;
171 *_strp_msg(skb
) = *_strp_msg(head
);
172 strp
->skb_nextp
= &head
->next
;
173 skb_shinfo(skb
)->frag_list
= head
;
174 strp
->skb_head
= skb
;
178 &skb_shinfo(head
)->frag_list
;
183 while (eaten
< orig_len
) {
184 /* Always clone since we will consume something */
185 skb
= skb_clone(orig_skb
, GFP_ATOMIC
);
187 STRP_STATS_INCR(strp
->stats
.mem_fail
);
188 desc
->error
= -ENOMEM
;
192 cand_len
= orig_len
- eaten
;
194 head
= strp
->skb_head
;
197 strp
->skb_head
= head
;
198 /* Will set skb_nextp on next packet if needed */
199 strp
->skb_nextp
= NULL
;
200 stm
= _strp_msg(head
);
201 memset(stm
, 0, sizeof(*stm
));
202 stm
->strp
.offset
= orig_offset
+ eaten
;
204 /* Unclone since we may be appending to an skb that we
205 * already share a frag_list with.
207 err
= skb_unclone(skb
, GFP_ATOMIC
);
209 STRP_STATS_INCR(strp
->stats
.mem_fail
);
214 stm
= _strp_msg(head
);
215 *strp
->skb_nextp
= skb
;
216 strp
->skb_nextp
= &skb
->next
;
217 head
->data_len
+= skb
->len
;
218 head
->len
+= skb
->len
;
219 head
->truesize
+= skb
->truesize
;
222 if (!stm
->strp
.full_len
) {
225 len
= (*strp
->cb
.parse_msg
)(strp
, head
);
228 /* Need more header to determine length */
229 if (!stm
->accum_len
) {
230 /* Start RX timer for new message */
231 strp_start_timer(strp
, timeo
);
233 stm
->accum_len
+= cand_len
;
235 STRP_STATS_INCR(strp
->stats
.need_more_hdr
);
236 WARN_ON(eaten
!= orig_len
);
238 } else if (len
< 0) {
239 if (len
== -ESTRPIPE
&& stm
->accum_len
) {
241 strp
->unrecov_intr
= 1;
243 strp
->interrupted
= 1;
245 strp_parser_err(strp
, len
, desc
);
247 } else if (len
> max_msg_size
) {
248 /* Message length exceeds maximum allowed */
249 STRP_STATS_INCR(strp
->stats
.msg_too_big
);
250 strp_parser_err(strp
, -EMSGSIZE
, desc
);
252 } else if (len
<= (ssize_t
)head
->len
-
253 skb
->len
- stm
->strp
.offset
) {
254 /* Length must be into new skb (and also
257 STRP_STATS_INCR(strp
->stats
.bad_hdr_len
);
258 strp_parser_err(strp
, -EPROTO
, desc
);
262 stm
->strp
.full_len
= len
;
265 extra
= (ssize_t
)(stm
->accum_len
+ cand_len
) -
269 /* Message not complete yet. */
270 if (stm
->strp
.full_len
- stm
->accum_len
>
271 strp_peek_len(strp
)) {
272 /* Don't have the whole message in the socket
273 * buffer. Set strp->need_bytes to wait for
274 * the rest of the message. Also, set "early
275 * eaten" since we've already buffered the skb
276 * but don't consume yet per strp_read_sock.
279 if (!stm
->accum_len
) {
280 /* Start RX timer for new message */
281 strp_start_timer(strp
, timeo
);
284 stm
->accum_len
+= cand_len
;
286 strp
->need_bytes
= stm
->strp
.full_len
-
288 STRP_STATS_ADD(strp
->stats
.bytes
, cand_len
);
289 desc
->count
= 0; /* Stop reading socket */
292 stm
->accum_len
+= cand_len
;
294 WARN_ON(eaten
!= orig_len
);
298 /* Positive extra indicates ore bytes than needed for the
302 WARN_ON(extra
> cand_len
);
304 eaten
+= (cand_len
- extra
);
306 /* Hurray, we have a new message! */
307 cancel_delayed_work(&strp
->msg_timer_work
);
308 strp
->skb_head
= NULL
;
309 strp
->need_bytes
= 0;
310 STRP_STATS_INCR(strp
->stats
.msgs
);
312 /* Give skb to upper layer */
313 strp
->cb
.rcv_msg(strp
, head
);
315 if (unlikely(strp
->paused
)) {
316 /* Upper layer paused strp */
324 STRP_STATS_ADD(strp
->stats
.bytes
, eaten
);
329 int strp_process(struct strparser
*strp
, struct sk_buff
*orig_skb
,
330 unsigned int orig_offset
, size_t orig_len
,
331 size_t max_msg_size
, long timeo
)
333 read_descriptor_t desc
; /* Dummy arg to strp_recv */
335 desc
.arg
.data
= strp
;
337 return __strp_recv(&desc
, orig_skb
, orig_offset
, orig_len
,
338 max_msg_size
, timeo
);
340 EXPORT_SYMBOL_GPL(strp_process
);
342 static int strp_recv(read_descriptor_t
*desc
, struct sk_buff
*orig_skb
,
343 unsigned int orig_offset
, size_t orig_len
)
345 struct strparser
*strp
= (struct strparser
*)desc
->arg
.data
;
347 return __strp_recv(desc
, orig_skb
, orig_offset
, orig_len
,
348 strp
->sk
->sk_rcvbuf
, strp
->sk
->sk_rcvtimeo
);
351 static int default_read_sock_done(struct strparser
*strp
, int err
)
356 /* Called with lock held on lower socket */
357 static int strp_read_sock(struct strparser
*strp
)
359 struct socket
*sock
= strp
->sk
->sk_socket
;
360 read_descriptor_t desc
;
362 if (unlikely(!sock
|| !sock
->ops
|| !sock
->ops
->read_sock
))
365 desc
.arg
.data
= strp
;
367 desc
.count
= 1; /* give more than one skb per call */
369 /* sk should be locked here, so okay to do read_sock */
370 sock
->ops
->read_sock(strp
->sk
, &desc
, strp_recv
);
372 desc
.error
= strp
->cb
.read_sock_done(strp
, desc
.error
);
377 /* Lower sock lock held */
378 void strp_data_ready(struct strparser
*strp
)
380 if (unlikely(strp
->stopped
))
383 /* This check is needed to synchronize with do_strp_work.
384 * do_strp_work acquires a process lock (lock_sock) whereas
385 * the lock held here is bh_lock_sock. The two locks can be
386 * held by different threads at the same time, but bh_lock_sock
387 * allows a thread in BH context to safely check if the process
388 * lock is held. In this case, if the lock is held, queue work.
390 if (sock_owned_by_user_nocheck(strp
->sk
)) {
391 queue_work(strp_wq
, &strp
->work
);
398 if (strp
->need_bytes
) {
399 if (strp_peek_len(strp
) < strp
->need_bytes
)
403 if (strp_read_sock(strp
) == -ENOMEM
)
404 queue_work(strp_wq
, &strp
->work
);
406 EXPORT_SYMBOL_GPL(strp_data_ready
);
408 static void do_strp_work(struct strparser
*strp
)
410 read_descriptor_t rd_desc
;
412 /* We need the read lock to synchronize with strp_data_ready. We
413 * need the socket lock for calling strp_read_sock.
417 if (unlikely(strp
->stopped
))
423 rd_desc
.arg
.data
= strp
;
425 if (strp_read_sock(strp
) == -ENOMEM
)
426 queue_work(strp_wq
, &strp
->work
);
429 strp
->cb
.unlock(strp
);
432 static void strp_work(struct work_struct
*w
)
434 do_strp_work(container_of(w
, struct strparser
, work
));
437 static void strp_msg_timeout(struct work_struct
*w
)
439 struct strparser
*strp
= container_of(w
, struct strparser
,
440 msg_timer_work
.work
);
442 /* Message assembly timed out */
443 STRP_STATS_INCR(strp
->stats
.msg_timeouts
);
445 strp
->cb
.abort_parser(strp
, -ETIMEDOUT
);
446 strp
->cb
.unlock(strp
);
449 static void strp_sock_lock(struct strparser
*strp
)
454 static void strp_sock_unlock(struct strparser
*strp
)
456 release_sock(strp
->sk
);
459 int strp_init(struct strparser
*strp
, struct sock
*sk
,
460 const struct strp_callbacks
*cb
)
463 if (!cb
|| !cb
->rcv_msg
|| !cb
->parse_msg
)
466 /* The sk (sock) arg determines the mode of the stream parser.
468 * If the sock is set then the strparser is in receive callback mode.
469 * The upper layer calls strp_data_ready to kick receive processing
470 * and strparser calls the read_sock function on the socket to
473 * If the sock is not set then the strparser is in general mode.
474 * The upper layer calls strp_process for each skb to be parsed.
478 if (!cb
->lock
|| !cb
->unlock
)
482 memset(strp
, 0, sizeof(*strp
));
486 strp
->cb
.lock
= cb
->lock
? : strp_sock_lock
;
487 strp
->cb
.unlock
= cb
->unlock
? : strp_sock_unlock
;
488 strp
->cb
.rcv_msg
= cb
->rcv_msg
;
489 strp
->cb
.parse_msg
= cb
->parse_msg
;
490 strp
->cb
.read_sock_done
= cb
->read_sock_done
? : default_read_sock_done
;
491 strp
->cb
.abort_parser
= cb
->abort_parser
? : strp_abort_strp
;
493 INIT_DELAYED_WORK(&strp
->msg_timer_work
, strp_msg_timeout
);
494 INIT_WORK(&strp
->work
, strp_work
);
498 EXPORT_SYMBOL_GPL(strp_init
);
500 void strp_unpause(struct strparser
*strp
)
504 /* Sync setting paused with RX work */
507 queue_work(strp_wq
, &strp
->work
);
509 EXPORT_SYMBOL_GPL(strp_unpause
);
511 /* strp must already be stopped so that strp_recv will no longer be called.
512 * Note that strp_done is not called with the lower socket held.
514 void strp_done(struct strparser
*strp
)
516 WARN_ON(!strp
->stopped
);
518 cancel_delayed_work_sync(&strp
->msg_timer_work
);
519 cancel_work_sync(&strp
->work
);
521 if (strp
->skb_head
) {
522 kfree_skb(strp
->skb_head
);
523 strp
->skb_head
= NULL
;
526 EXPORT_SYMBOL_GPL(strp_done
);
528 void strp_stop(struct strparser
*strp
)
532 EXPORT_SYMBOL_GPL(strp_stop
);
534 void strp_check_rcv(struct strparser
*strp
)
536 queue_work(strp_wq
, &strp
->work
);
538 EXPORT_SYMBOL_GPL(strp_check_rcv
);
540 static int __init
strp_mod_init(void)
542 strp_wq
= create_singlethread_workqueue("kstrp");
547 static void __exit
strp_mod_exit(void)
549 destroy_workqueue(strp_wq
);
551 module_init(strp_mod_init
);
552 module_exit(strp_mod_exit
);
553 MODULE_LICENSE("GPL");