]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - net/packet/af_packet.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xen-netback: stop the VIF thread before unbinding IRQs
[mirror_ubuntu-zesty-kernel.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
5df0ddfb 91#include <linux/reciprocal_div.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
ce06b03e
DM		 240/* register_prot_hook must be invoked with the po->bind_lock held,
241 * or from a context in which asynchronous accesses to the packet
242 * socket is not possible (packet_create()).
243 */
244static void register_prot_hook(struct sock *sk)
245{
246 struct packet_sock *po = pkt_sk(sk);
247 if (!po->running) {
dc99f600
DM		 248 if (po->fanout)
249 __fanout_link(sk, po);
250 else
251 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 252 sock_hold(sk);
253 po->running = 1;
254 }
255}
256
257/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
258 * held. If the sync parameter is true, we will temporarily drop
259 * the po->bind_lock and do a synchronize_net to make sure no
260 * asynchronous packet processing paths still refer to the elements
261 * of po->prot_hook. If the sync parameter is false, it is the
262 * callers responsibility to take care of this.
263 */
264static void __unregister_prot_hook(struct sock *sk, bool sync)
265{
266 struct packet_sock *po = pkt_sk(sk);
267
268 po->running = 0;
dc99f600
DM		 269 if (po->fanout)
270 __fanout_unlink(sk, po);
271 else
272 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 273 __sock_put(sk);
274
275 if (sync) {
276 spin_unlock(&po->bind_lock);
277 synchronize_net();
278 spin_lock(&po->bind_lock);
279 }
280}
281
282static void unregister_prot_hook(struct sock *sk, bool sync)
283{
284 struct packet_sock *po = pkt_sk(sk);
285
286 if (po->running)
287 __unregister_prot_hook(sk, sync);
288}
289
f6dafa95 290static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 291{
292 if (is_vmalloc_addr(addr))
293 return vmalloc_to_page(addr);
294 return virt_to_page(addr);
295}
296
69e3c75f 297static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 298{
184f489e 299 union tpacket_uhdr h;
1da177e4 300
69e3c75f 301 h.raw = frame;
bbd6ef87
PM		 302 switch (po->tp_version) {
303 case TPACKET_V1:
69e3c75f 304 h.h1->tp_status = status;
0af55bb5 305 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 306 break;
307 case TPACKET_V2:
69e3c75f 308 h.h2->tp_status = status;
0af55bb5 309 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 310 break;
f6fb8f10 311 case TPACKET_V3:
69e3c75f 312 default:
f6fb8f10 313 WARN(1, "TPACKET version not supported.\n");
69e3c75f 314 BUG();
bbd6ef87 315 }
69e3c75f
JB		 316
317 smp_wmb();
bbd6ef87
PM		 318}
319
69e3c75f 320static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 321{
184f489e 322 union tpacket_uhdr h;
bbd6ef87 323
69e3c75f
JB		 324 smp_rmb();
325
bbd6ef87
PM		 326 h.raw = frame;
327 switch (po->tp_version) {
328 case TPACKET_V1:
0af55bb5 329 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 330 return h.h1->tp_status;
bbd6ef87 331 case TPACKET_V2:
0af55bb5 332 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 333 return h.h2->tp_status;
f6fb8f10 334 case TPACKET_V3:
69e3c75f 335 default:
f6fb8f10 336 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 337 BUG();
338 return 0;
bbd6ef87 339 }
1da177e4 340}
69e3c75f 341
b9c32fb2
DB		 342static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
343 unsigned int flags)
7a51384c
DB		 344{
345 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
346
347 if (shhwtstamps) {
348 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
349 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 350 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 351 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
352 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 353 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 354 }
355
356 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 357 return TP_STATUS_TS_SOFTWARE;
7a51384c 358
b9c32fb2 359 return 0;
7a51384c
DB		 360}
361
b9c32fb2
DB		 362static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
363 struct sk_buff *skb)
2e31396f
WB		 364{
365 union tpacket_uhdr h;
366 struct timespec ts;
b9c32fb2 367 __u32 ts_status;
2e31396f 368
b9c32fb2
DB		 369 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
370 return 0;
2e31396f
WB		 371
372 h.raw = frame;
373 switch (po->tp_version) {
374 case TPACKET_V1:
375 h.h1->tp_sec = ts.tv_sec;
376 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
377 break;
378 case TPACKET_V2:
379 h.h2->tp_sec = ts.tv_sec;
380 h.h2->tp_nsec = ts.tv_nsec;
381 break;
382 case TPACKET_V3:
383 default:
384 WARN(1, "TPACKET version not supported.\n");
385 BUG();
386 }
387
388 /* one flush is safe, as both fields always lie on the same cacheline */
389 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
390 smp_wmb();
b9c32fb2
DB		 391
392 return ts_status;
2e31396f
WB		 393}
394
69e3c75f
JB		 395static void *packet_lookup_frame(struct packet_sock *po,
396 struct packet_ring_buffer *rb,
397 unsigned int position,
398 int status)
399{
400 unsigned int pg_vec_pos, frame_offset;
184f489e 401 union tpacket_uhdr h;
69e3c75f
JB		 402
403 pg_vec_pos = position / rb->frames_per_block;
404 frame_offset = position % rb->frames_per_block;
405
0e3125c7
NH		 406 h.raw = rb->pg_vec[pg_vec_pos].buffer +
407 (frame_offset * rb->frame_size);
69e3c75f
JB		 408
409 if (status != __packet_get_status(po, h.raw))
410 return NULL;
411
412 return h.raw;
413}
414
eea49cc9 415static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 416 struct packet_ring_buffer *rb,
417 int status)
418{
419 return packet_lookup_frame(po, rb, rb->head, status);
420}
421
bc59ba39 422static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 423{
424 del_timer_sync(&pkc->retire_blk_timer);
425}
426
427static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
428 int tx_ring,
429 struct sk_buff_head *rb_queue)
430{
bc59ba39 431 struct tpacket_kbdq_core *pkc;
f6fb8f10 432
433 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
434
435 spin_lock(&rb_queue->lock);
436 pkc->delete_blk_timer = 1;
437 spin_unlock(&rb_queue->lock);
438
439 prb_del_retire_blk_timer(pkc);
440}
441
442static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 443 struct tpacket_kbdq_core *pkc,
f6fb8f10 444 void (*func) (unsigned long))
445{
446 init_timer(&pkc->retire_blk_timer);
447 pkc->retire_blk_timer.data = (long)po;
448 pkc->retire_blk_timer.function = func;
449 pkc->retire_blk_timer.expires = jiffies;
450}
451
452static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
453{
bc59ba39 454 struct tpacket_kbdq_core *pkc;
f6fb8f10 455
456 if (tx_ring)
457 BUG();
458
459 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
460 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
461}
462
463static int prb_calc_retire_blk_tmo(struct packet_sock *po,
464 int blk_size_in_bytes)
465{
466 struct net_device *dev;
467 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 468 struct ethtool_cmd ecmd;
469 int err;
e440cf2c 470 u32 speed;
f6fb8f10 471
4bc71cb9
JP		 472 rtnl_lock();
473 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
474 if (unlikely(!dev)) {
475 rtnl_unlock();
f6fb8f10 476 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 477 }
478 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 479 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 480 rtnl_unlock();
481 if (!err) {
4bc71cb9
JP		 482 /*
483 * If the link speed is so slow you don't really
484 * need to worry about perf anyways
485 */
e440cf2c 486 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 487 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 488 } else {
489 msec = 1;
490 div = speed / 1000;
f6fb8f10 491 }
492 }
493
494 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
495
496 if (div)
497 mbits /= div;
498
499 tmo = mbits * msec;
500
501 if (div)
502 return tmo+1;
503 return tmo;
504}
505
bc59ba39 506static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 507 union tpacket_req_u *req_u)
508{
509 p1->feature_req_word = req_u->req3.tp_feature_req_word;
510}
511
512static void init_prb_bdqc(struct packet_sock *po,
513 struct packet_ring_buffer *rb,
514 struct pgv *pg_vec,
515 union tpacket_req_u *req_u, int tx_ring)
516{
bc59ba39 517 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
518 struct tpacket_block_desc *pbd;
f6fb8f10 519
520 memset(p1, 0x0, sizeof(*p1));
521
522 p1->knxt_seq_num = 1;
523 p1->pkbdq = pg_vec;
bc59ba39 524 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 525 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 526 p1->kblk_size = req_u->req3.tp_block_size;
527 p1->knum_blocks = req_u->req3.tp_block_nr;
528 p1->hdrlen = po->tp_hdrlen;
529 p1->version = po->tp_version;
530 p1->last_kactive_blk_num = 0;
ee80fbf3 531 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 532 if (req_u->req3.tp_retire_blk_tov)
533 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
534 else
535 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
536 req_u->req3.tp_block_size);
537 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
538 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
539
540 prb_init_ft_ops(p1, req_u);
541 prb_setup_retire_blk_timer(po, tx_ring);
542 prb_open_block(p1, pbd);
543}
544
545/* Do NOT update the last_blk_num first.
546 * Assumes sk_buff_head lock is held.
547 */
bc59ba39 548static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 549{
550 mod_timer(&pkc->retire_blk_timer,
551 jiffies + pkc->tov_in_jiffies);
552 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
553}
554
555/*
556 * Timer logic:
557 * 1) We refresh the timer only when we open a block.
558 * By doing this we don't waste cycles refreshing the timer
559 * on packet-by-packet basis.
560 *
561 * With a 1MB block-size, on a 1Gbps line, it will take
562 * i) ~8 ms to fill a block + ii) memcpy etc.
563 * In this cut we are not accounting for the memcpy time.
564 *
565 * So, if the user sets the 'tmo' to 10ms then the timer
566 * will never fire while the block is still getting filled
567 * (which is what we want). However, the user could choose
568 * to close a block early and that's fine.
569 *
570 * But when the timer does fire, we check whether or not to refresh it.
571 * Since the tmo granularity is in msecs, it is not too expensive
572 * to refresh the timer, lets say every '8' msecs.
573 * Either the user can set the 'tmo' or we can derive it based on
574 * a) line-speed and b) block-size.
575 * prb_calc_retire_blk_tmo() calculates the tmo.
576 *
577 */
578static void prb_retire_rx_blk_timer_expired(unsigned long data)
579{
580 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 581 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 582 unsigned int frozen;
bc59ba39 583 struct tpacket_block_desc *pbd;
f6fb8f10 584
585 spin_lock(&po->sk.sk_receive_queue.lock);
586
587 frozen = prb_queue_frozen(pkc);
588 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
589
590 if (unlikely(pkc->delete_blk_timer))
591 goto out;
592
593 /* We only need to plug the race when the block is partially filled.
594 * tpacket_rcv:
595 * lock(); increment BLOCK_NUM_PKTS; unlock()
596 * copy_bits() is in progress ...
597 * timer fires on other cpu:
598 * we can't retire the current block because copy_bits
599 * is in progress.
600 *
601 */
602 if (BLOCK_NUM_PKTS(pbd)) {
603 while (atomic_read(&pkc->blk_fill_in_prog)) {
604 /* Waiting for skb_copy_bits to finish... */
605 cpu_relax();
606 }
607 }
608
609 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
610 if (!frozen) {
611 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
612 if (!prb_dispatch_next_block(pkc, po))
613 goto refresh_timer;
614 else
615 goto out;
616 } else {
617 /* Case 1. Queue was frozen because user-space was
618 * lagging behind.
619 */
620 if (prb_curr_blk_in_use(pkc, pbd)) {
621 /*
622 * Ok, user-space is still behind.
623 * So just refresh the timer.
624 */
625 goto refresh_timer;
626 } else {
627 /* Case 2. queue was frozen,user-space caught up,
628 * now the link went idle && the timer fired.
629 * We don't have a block to close.So we open this
630 * block and restart the timer.
631 * opening a block thaws the queue,restarts timer
632 * Thawing/timer-refresh is a side effect.
633 */
634 prb_open_block(pkc, pbd);
635 goto out;
636 }
637 }
638 }
639
640refresh_timer:
641 _prb_refresh_rx_retire_blk_timer(pkc);
642
643out:
644 spin_unlock(&po->sk.sk_receive_queue.lock);
645}
646
eea49cc9 647static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 648 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 649{
650 /* Flush everything minus the block header */
651
652#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
653 u8 *start, *end;
654
655 start = (u8 *)pbd1;
656
657 /* Skip the block header(we know header WILL fit in 4K) */
658 start += PAGE_SIZE;
659
660 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
661 for (; start < end; start += PAGE_SIZE)
662 flush_dcache_page(pgv_to_page(start));
663
664 smp_wmb();
665#endif
666
667 /* Now update the block status. */
668
669 BLOCK_STATUS(pbd1) = status;
670
671 /* Flush the block header */
672
673#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
674 start = (u8 *)pbd1;
675 flush_dcache_page(pgv_to_page(start));
676
677 smp_wmb();
678#endif
679}
680
681/*
682 * Side effect:
683 *
684 * 1) flush the block
685 * 2) Increment active_blk_num
686 *
687 * Note:We DONT refresh the timer on purpose.
688 * Because almost always the next block will be opened.
689 */
bc59ba39 690static void prb_close_block(struct tpacket_kbdq_core *pkc1,
691 struct tpacket_block_desc *pbd1,
f6fb8f10 692 struct packet_sock *po, unsigned int stat)
693{
694 __u32 status = TP_STATUS_USER | stat;
695
696 struct tpacket3_hdr *last_pkt;
bc59ba39 697 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 698
ee80fbf3 699 if (po->stats.stats3.tp_drops)
f6fb8f10 700 status |= TP_STATUS_LOSING;
701
702 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
703 last_pkt->tp_next_offset = 0;
704
705 /* Get the ts of the last pkt */
706 if (BLOCK_NUM_PKTS(pbd1)) {
707 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
708 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
709 } else {
710 /* Ok, we tmo'd - so get the current time */
711 struct timespec ts;
712 getnstimeofday(&ts);
713 h1->ts_last_pkt.ts_sec = ts.tv_sec;
714 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
715 }
716
717 smp_wmb();
718
719 /* Flush the block */
720 prb_flush_block(pkc1, pbd1, status);
721
722 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
723}
724
eea49cc9 725static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 726{
727 pkc->reset_pending_on_curr_blk = 0;
728}
729
730/*
731 * Side effect of opening a block:
732 *
733 * 1) prb_queue is thawed.
734 * 2) retire_blk_timer is refreshed.
735 *
736 */
bc59ba39 737static void prb_open_block(struct tpacket_kbdq_core *pkc1,
738 struct tpacket_block_desc *pbd1)
f6fb8f10 739{
740 struct timespec ts;
bc59ba39 741 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 742
743 smp_rmb();
744
8da3056c
DB		 745 /* We could have just memset this but we will lose the
746 * flexibility of making the priv area sticky
747 */
f6fb8f10 748
8da3056c
DB		 749 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
750 BLOCK_NUM_PKTS(pbd1) = 0;
751 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 752
8da3056c
DB		 753 getnstimeofday(&ts);
754
755 h1->ts_first_pkt.ts_sec = ts.tv_sec;
756 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 757
8da3056c
DB		 758 pkc1->pkblk_start = (char *)pbd1;
759 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
760
761 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
762 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
763
764 pbd1->version = pkc1->version;
765 pkc1->prev = pkc1->nxt_offset;
766 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
767
768 prb_thaw_queue(pkc1);
769 _prb_refresh_rx_retire_blk_timer(pkc1);
770
771 smp_wmb();
f6fb8f10 772}
773
774/*
775 * Queue freeze logic:
776 * 1) Assume tp_block_nr = 8 blocks.
777 * 2) At time 't0', user opens Rx ring.
778 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
779 * 4) user-space is either sleeping or processing block '0'.
780 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
781 * it will close block-7,loop around and try to fill block '0'.
782 * call-flow:
783 * __packet_lookup_frame_in_block
784 * prb_retire_current_block()
785 * prb_dispatch_next_block()
786 * |->(BLOCK_STATUS == USER) evaluates to true
787 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
788 * 6) Now there are two cases:
789 * 6.1) Link goes idle right after the queue is frozen.
790 * But remember, the last open_block() refreshed the timer.
791 * When this timer expires,it will refresh itself so that we can
792 * re-open block-0 in near future.
793 * 6.2) Link is busy and keeps on receiving packets. This is a simple
794 * case and __packet_lookup_frame_in_block will check if block-0
795 * is free and can now be re-used.
796 */
eea49cc9 797static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 798 struct packet_sock *po)
799{
800 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 801 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 802}
803
804#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
805
806/*
807 * If the next block is free then we will dispatch it
808 * and return a good offset.
809 * Else, we will freeze the queue.
810 * So, caller must check the return value.
811 */
bc59ba39 812static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 813 struct packet_sock *po)
814{
bc59ba39 815 struct tpacket_block_desc *pbd;
f6fb8f10 816
817 smp_rmb();
818
819 /* 1. Get current block num */
820 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
821
822 /* 2. If this block is currently in_use then freeze the queue */
823 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
824 prb_freeze_queue(pkc, po);
825 return NULL;
826 }
827
828 /*
829 * 3.
830 * open this block and return the offset where the first packet
831 * needs to get stored.
832 */
833 prb_open_block(pkc, pbd);
834 return (void *)pkc->nxt_offset;
835}
836
bc59ba39 837static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 838 struct packet_sock *po, unsigned int status)
839{
bc59ba39 840 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 841
842 /* retire/close the current block */
843 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
844 /*
845 * Plug the case where copy_bits() is in progress on
846 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
847 * have space to copy the pkt in the current block and
848 * called prb_retire_current_block()
849 *
850 * We don't need to worry about the TMO case because
851 * the timer-handler already handled this case.
852 */
853 if (!(status & TP_STATUS_BLK_TMO)) {
854 while (atomic_read(&pkc->blk_fill_in_prog)) {
855 /* Waiting for skb_copy_bits to finish... */
856 cpu_relax();
857 }
858 }
859 prb_close_block(pkc, pbd, po, status);
860 return;
861 }
f6fb8f10 862}
863
eea49cc9 864static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 865 struct tpacket_block_desc *pbd)
f6fb8f10 866{
867 return TP_STATUS_USER & BLOCK_STATUS(pbd);
868}
869
eea49cc9 870static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 871{
872 return pkc->reset_pending_on_curr_blk;
873}
874
eea49cc9 875static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 876{
bc59ba39 877 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 878 atomic_dec(&pkc->blk_fill_in_prog);
879}
880
eea49cc9 881static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 882 struct tpacket3_hdr *ppd)
883{
884 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
885}
886
eea49cc9 887static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 888 struct tpacket3_hdr *ppd)
889{
890 ppd->hv1.tp_rxhash = 0;
891}
892
eea49cc9 893static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 894 struct tpacket3_hdr *ppd)
895{
896 if (vlan_tx_tag_present(pkc->skb)) {
897 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
898 ppd->tp_status = TP_STATUS_VLAN_VALID;
899 } else {
9e67030a 900 ppd->hv1.tp_vlan_tci = 0;
901 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 902 }
903}
904
bc59ba39 905static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 906 struct tpacket3_hdr *ppd)
907{
908 prb_fill_vlan_info(pkc, ppd);
909
910 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
911 prb_fill_rxhash(pkc, ppd);
912 else
913 prb_clear_rxhash(pkc, ppd);
914}
915
eea49cc9 916static void prb_fill_curr_block(char *curr,
bc59ba39 917 struct tpacket_kbdq_core *pkc,
918 struct tpacket_block_desc *pbd,
f6fb8f10 919 unsigned int len)
920{
921 struct tpacket3_hdr *ppd;
922
923 ppd = (struct tpacket3_hdr *)curr;
924 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
925 pkc->prev = curr;
926 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
927 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
928 BLOCK_NUM_PKTS(pbd) += 1;
929 atomic_inc(&pkc->blk_fill_in_prog);
930 prb_run_all_ft_ops(pkc, ppd);
931}
932
933/* Assumes caller has the sk->rx_queue.lock */
934static void *__packet_lookup_frame_in_block(struct packet_sock *po,
935 struct sk_buff *skb,
936 int status,
937 unsigned int len
938 )
939{
bc59ba39 940 struct tpacket_kbdq_core *pkc;
941 struct tpacket_block_desc *pbd;
f6fb8f10 942 char *curr, *end;
943
e3192690 944 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 945 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
946
947 /* Queue is frozen when user space is lagging behind */
948 if (prb_queue_frozen(pkc)) {
949 /*
950 * Check if that last block which caused the queue to freeze,
951 * is still in_use by user-space.
952 */
953 if (prb_curr_blk_in_use(pkc, pbd)) {
954 /* Can't record this packet */
955 return NULL;
956 } else {
957 /*
958 * Ok, the block was released by user-space.
959 * Now let's open that block.
960 * opening a block also thaws the queue.
961 * Thawing is a side effect.
962 */
963 prb_open_block(pkc, pbd);
964 }
965 }
966
967 smp_mb();
968 curr = pkc->nxt_offset;
969 pkc->skb = skb;
e3192690 970 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 971
972 /* first try the current block */
973 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
974 prb_fill_curr_block(curr, pkc, pbd, len);
975 return (void *)curr;
976 }
977
978 /* Ok, close the current block */
979 prb_retire_current_block(pkc, po, 0);
980
981 /* Now, try to dispatch the next block */
982 curr = (char *)prb_dispatch_next_block(pkc, po);
983 if (curr) {
984 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
985 prb_fill_curr_block(curr, pkc, pbd, len);
986 return (void *)curr;
987 }
988
989 /*
990 * No free blocks are available.user_space hasn't caught up yet.
991 * Queue was just frozen and now this packet will get dropped.
992 */
993 return NULL;
994}
995
eea49cc9 996static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 997 struct sk_buff *skb,
998 int status, unsigned int len)
999{
1000 char *curr = NULL;
1001 switch (po->tp_version) {
1002 case TPACKET_V1:
1003 case TPACKET_V2:
1004 curr = packet_lookup_frame(po, &po->rx_ring,
1005 po->rx_ring.head, status);
1006 return curr;
1007 case TPACKET_V3:
1008 return __packet_lookup_frame_in_block(po, skb, status, len);
1009 default:
1010 WARN(1, "TPACKET version not supported\n");
1011 BUG();
99aa3473 1012 return NULL;
f6fb8f10 1013 }
1014}
1015
eea49cc9 1016static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1017 struct packet_ring_buffer *rb,
77f65ebd 1018 unsigned int idx,
f6fb8f10 1019 int status)
1020{
bc59ba39 1021 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1022 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1023
1024 if (status != BLOCK_STATUS(pbd))
1025 return NULL;
1026 return pbd;
1027}
1028
eea49cc9 1029static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1030{
1031 unsigned int prev;
1032 if (rb->prb_bdqc.kactive_blk_num)
1033 prev = rb->prb_bdqc.kactive_blk_num-1;
1034 else
1035 prev = rb->prb_bdqc.knum_blocks-1;
1036 return prev;
1037}
1038
1039/* Assumes caller has held the rx_queue.lock */
eea49cc9 1040static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1041 struct packet_ring_buffer *rb,
1042 int status)
1043{
1044 unsigned int previous = prb_previous_blk_num(rb);
1045 return prb_lookup_block(po, rb, previous, status);
1046}
1047
eea49cc9 1048static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1049 struct packet_ring_buffer *rb,
1050 int status)
1051{
1052 if (po->tp_version <= TPACKET_V2)
1053 return packet_previous_frame(po, rb, status);
1054
1055 return __prb_previous_block(po, rb, status);
1056}
1057
eea49cc9 1058static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1059 struct packet_ring_buffer *rb)
1060{
1061 switch (po->tp_version) {
1062 case TPACKET_V1:
1063 case TPACKET_V2:
1064 return packet_increment_head(rb);
1065 case TPACKET_V3:
1066 default:
1067 WARN(1, "TPACKET version not supported.\n");
1068 BUG();
1069 return;
1070 }
1071}
1072
eea49cc9 1073static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1074 struct packet_ring_buffer *rb,
1075 int status)
1076{
1077 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1078 return packet_lookup_frame(po, rb, previous, status);
1079}
1080
eea49cc9 1081static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1082{
1083 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1084}
1085
77f65ebd
WB		 1086static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1087{
1088 struct sock *sk = &po->sk;
1089 bool has_room;
1090
1091 if (po->prot_hook.func != tpacket_rcv)
1092 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1093 <= sk->sk_rcvbuf;
1094
1095 spin_lock(&sk->sk_receive_queue.lock);
1096 if (po->tp_version == TPACKET_V3)
1097 has_room = prb_lookup_block(po, &po->rx_ring,
1098 po->rx_ring.prb_bdqc.kactive_blk_num,
1099 TP_STATUS_KERNEL);
1100 else
1101 has_room = packet_lookup_frame(po, &po->rx_ring,
1102 po->rx_ring.head,
1103 TP_STATUS_KERNEL);
1104 spin_unlock(&sk->sk_receive_queue.lock);
1105
1106 return has_room;
1107}
1108
1da177e4
LT		 1109static void packet_sock_destruct(struct sock *sk)
1110{
ed85b565
RC		 1111 skb_queue_purge(&sk->sk_error_queue);
1112
547b792c
IJ		 1113 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1114 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1115
1116 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1117 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1118 return;
1119 }
1120
17ab56a2 1121 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1122}
1123
dc99f600
DM		 1124static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1125{
1126 int x = atomic_read(&f->rr_cur) + 1;
1127
1128 if (x >= num)
1129 x = 0;
1130
1131 return x;
1132}
1133
77f65ebd
WB		 1134static unsigned int fanout_demux_hash(struct packet_fanout *f,
1135 struct sk_buff *skb,
1136 unsigned int num)
dc99f600 1137{
f55d112e 1138 return reciprocal_divide(skb->rxhash, num);
dc99f600
DM		 1139}
1140
77f65ebd
WB		 1141static unsigned int fanout_demux_lb(struct packet_fanout *f,
1142 struct sk_buff *skb,
1143 unsigned int num)
dc99f600
DM		 1144{
1145 int cur, old;
1146
1147 cur = atomic_read(&f->rr_cur);
1148 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1149 fanout_rr_next(f, num))) != cur)
1150 cur = old;
77f65ebd
WB		 1151 return cur;
1152}
1153
1154static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1155 struct sk_buff *skb,
1156 unsigned int num)
1157{
1158 return smp_processor_id() % num;
dc99f600
DM		 1159}
1160
5df0ddfb
DB		 1161static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1162 struct sk_buff *skb,
1163 unsigned int num)
1164{
1165 return reciprocal_divide(prandom_u32(), num);
1166}
1167
77f65ebd
WB		 1168static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1169 struct sk_buff *skb,
1170 unsigned int idx, unsigned int skip,
1171 unsigned int num)
95ec3eb4 1172{
77f65ebd 1173 unsigned int i, j;
95ec3eb4 1174
77f65ebd
WB		 1175 i = j = min_t(int, f->next[idx], num - 1);
1176 do {
1177 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1178 if (i != j)
1179 f->next[idx] = i;
1180 return i;
1181 }
1182 if (++i == num)
1183 i = 0;
1184 } while (i != j);
1185
1186 return idx;
1187}
1188
1189static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1190{
1191 return f->flags & (flag >> 8);
95ec3eb4
DM		 1192}
1193
95ec3eb4
DM		 1194static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1195 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1196{
1197 struct packet_fanout *f = pt->af_packet_priv;
1198 unsigned int num = f->num_members;
1199 struct packet_sock *po;
77f65ebd 1200 unsigned int idx;
dc99f600
DM		 1201
1202 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1203 !num) {
1204 kfree_skb(skb);
1205 return 0;
1206 }
1207
95ec3eb4
DM		 1208 switch (f->type) {
1209 case PACKET_FANOUT_HASH:
1210 default:
77f65ebd 1211 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1212 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1213 if (!skb)
1214 return 0;
1215 }
1216 skb_get_rxhash(skb);
77f65ebd 1217 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1218 break;
1219 case PACKET_FANOUT_LB:
77f65ebd 1220 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1221 break;
1222 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1223 idx = fanout_demux_cpu(f, skb, num);
1224 break;
5df0ddfb
DB		 1225 case PACKET_FANOUT_RND:
1226 idx = fanout_demux_rnd(f, skb, num);
1227 break;
77f65ebd
WB		 1228 case PACKET_FANOUT_ROLLOVER:
1229 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1230 break;
dc99f600
DM		 1231 }
1232
77f65ebd
WB		 1233 po = pkt_sk(f->arr[idx]);
1234 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1235 unlikely(!packet_rcv_has_room(po, skb))) {
1236 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1237 po = pkt_sk(f->arr[idx]);
1238 }
dc99f600
DM		 1239
1240 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1241}
1242
fff3321d
PE		 1243DEFINE_MUTEX(fanout_mutex);
1244EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1245static LIST_HEAD(fanout_list);
1246
1247static void __fanout_link(struct sock *sk, struct packet_sock *po)
1248{
1249 struct packet_fanout *f = po->fanout;
1250
1251 spin_lock(&f->lock);
1252 f->arr[f->num_members] = sk;
1253 smp_wmb();
1254 f->num_members++;
1255 spin_unlock(&f->lock);
1256}
1257
1258static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1259{
1260 struct packet_fanout *f = po->fanout;
1261 int i;
1262
1263 spin_lock(&f->lock);
1264 for (i = 0; i < f->num_members; i++) {
1265 if (f->arr[i] == sk)
1266 break;
1267 }
1268 BUG_ON(i >= f->num_members);
1269 f->arr[i] = f->arr[f->num_members - 1];
1270 f->num_members--;
1271 spin_unlock(&f->lock);
1272}
1273
a0dfb263 1274static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1275{
1276 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1277 return true;
1278
1279 return false;
1280}
1281
7736d33f 1282static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1283{
1284 struct packet_sock *po = pkt_sk(sk);
1285 struct packet_fanout *f, *match;
7736d33f 1286 u8 type = type_flags & 0xff;
77f65ebd 1287 u8 flags = type_flags >> 8;
dc99f600
DM		 1288 int err;
1289
1290 switch (type) {
77f65ebd
WB		 1291 case PACKET_FANOUT_ROLLOVER:
1292 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1293 return -EINVAL;
dc99f600
DM		 1294 case PACKET_FANOUT_HASH:
1295 case PACKET_FANOUT_LB:
95ec3eb4 1296 case PACKET_FANOUT_CPU:
5df0ddfb 1297 case PACKET_FANOUT_RND:
dc99f600
DM		 1298 break;
1299 default:
1300 return -EINVAL;
1301 }
1302
1303 if (!po->running)
1304 return -EINVAL;
1305
1306 if (po->fanout)
1307 return -EALREADY;
1308
1309 mutex_lock(&fanout_mutex);
1310 match = NULL;
1311 list_for_each_entry(f, &fanout_list, list) {
1312 if (f->id == id &&
1313 read_pnet(&f->net) == sock_net(sk)) {
1314 match = f;
1315 break;
1316 }
1317 }
afe62c68 1318 err = -EINVAL;
77f65ebd 1319 if (match && match->flags != flags)
afe62c68 1320 goto out;
dc99f600 1321 if (!match) {
afe62c68 1322 err = -ENOMEM;
dc99f600 1323 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1324 if (!match)
1325 goto out;
1326 write_pnet(&match->net, sock_net(sk));
1327 match->id = id;
1328 match->type = type;
77f65ebd 1329 match->flags = flags;
afe62c68
ED		 1330 atomic_set(&match->rr_cur, 0);
1331 INIT_LIST_HEAD(&match->list);
1332 spin_lock_init(&match->lock);
1333 atomic_set(&match->sk_ref, 0);
1334 match->prot_hook.type = po->prot_hook.type;
1335 match->prot_hook.dev = po->prot_hook.dev;
1336 match->prot_hook.func = packet_rcv_fanout;
1337 match->prot_hook.af_packet_priv = match;
c0de08d0 1338 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1339 dev_add_pack(&match->prot_hook);
1340 list_add(&match->list, &fanout_list);
dc99f600 1341 }
afe62c68
ED		 1342 err = -EINVAL;
1343 if (match->type == type &&
1344 match->prot_hook.type == po->prot_hook.type &&
1345 match->prot_hook.dev == po->prot_hook.dev) {
1346 err = -ENOSPC;
1347 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1348 __dev_remove_pack(&po->prot_hook);
1349 po->fanout = match;
1350 atomic_inc(&match->sk_ref);
1351 __fanout_link(sk, po);
1352 err = 0;
dc99f600
DM		 1353 }
1354 }
afe62c68 1355out:
dc99f600
DM		 1356 mutex_unlock(&fanout_mutex);
1357 return err;
1358}
1359
1360static void fanout_release(struct sock *sk)
1361{
1362 struct packet_sock *po = pkt_sk(sk);
1363 struct packet_fanout *f;
1364
1365 f = po->fanout;
1366 if (!f)
1367 return;
1368
fff3321d 1369 mutex_lock(&fanout_mutex);
dc99f600
DM		 1370 po->fanout = NULL;
1371
dc99f600
DM		 1372 if (atomic_dec_and_test(&f->sk_ref)) {
1373 list_del(&f->list);
1374 dev_remove_pack(&f->prot_hook);
1375 kfree(f);
1376 }
1377 mutex_unlock(&fanout_mutex);
1378}
1da177e4 1379
90ddc4f0 1380static const struct proto_ops packet_ops;
1da177e4 1381
90ddc4f0 1382static const struct proto_ops packet_ops_spkt;
1da177e4 1383
40d4e3df
ED		 1384static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1385 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1386{
1387 struct sock *sk;
1388 struct sockaddr_pkt *spkt;
1389
1390 /*
1391 * When we registered the protocol we saved the socket in the data
1392 * field for just this event.
1393 */
1394
1395 sk = pt->af_packet_priv;
1ce4f28b 1396
1da177e4
LT		 1397 /*
1398 * Yank back the headers [hope the device set this
1399 * right or kerboom...]
1400 *
1401 * Incoming packets have ll header pulled,
1402 * push it back.
1403 *
98e399f8 1404 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1405 * so that this procedure is noop.
1406 */
1407
1408 if (skb->pkt_type == PACKET_LOOPBACK)
1409 goto out;
1410
09ad9bc7 1411 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1412 goto out;
1413
40d4e3df
ED		 1414 skb = skb_share_check(skb, GFP_ATOMIC);
1415 if (skb == NULL)
1da177e4
LT		 1416 goto oom;
1417
1418 /* drop any routing info */
adf30907 1419 skb_dst_drop(skb);
1da177e4 1420
84531c24
PO		 1421 /* drop conntrack reference */
1422 nf_reset(skb);
1423
ffbc6111 1424 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1425
98e399f8 1426 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1427
1428 /*
1429 * The SOCK_PACKET socket receives _all_ frames.
1430 */
1431
1432 spkt->spkt_family = dev->type;
1433 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1434 spkt->spkt_protocol = skb->protocol;
1435
1436 /*
1437 * Charge the memory to the socket. This is done specifically
1438 * to prevent sockets using all the memory up.
1439 */
1440
40d4e3df 1441 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1442 return 0;
1443
1444out:
1445 kfree_skb(skb);
1446oom:
1447 return 0;
1448}
1449
1450
1451/*
1452 * Output a raw packet to a device layer. This bypasses all the other
1453 * protocol layers and you must therefore supply it with a complete frame
1454 */
1ce4f28b 1455
1da177e4
LT		 1456static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1457 struct msghdr *msg, size_t len)
1458{
1459 struct sock *sk = sock->sk;
40d4e3df 1460 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1461 struct sk_buff *skb = NULL;
1da177e4 1462 struct net_device *dev;
40d4e3df 1463 __be16 proto = 0;
1da177e4 1464 int err;
3bdc0eba 1465 int extra_len = 0;
1ce4f28b 1466
1da177e4 1467 /*
1ce4f28b 1468 * Get and verify the address.
1da177e4
LT		 1469 */
1470
40d4e3df 1471 if (saddr) {
1da177e4 1472 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1473 return -EINVAL;
1474 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1475 proto = saddr->spkt_protocol;
1476 } else
1477 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1478
1479 /*
1ce4f28b 1480 * Find the device first to size check it
1da177e4
LT		 1481 */
1482
de74e92a 1483 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1484retry:
654d1f8a
ED		 1485 rcu_read_lock();
1486 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1487 err = -ENODEV;
1488 if (dev == NULL)
1489 goto out_unlock;
1ce4f28b 1490
d5e76b0a
DM		 1491 err = -ENETDOWN;
1492 if (!(dev->flags & IFF_UP))
1493 goto out_unlock;
1494
1da177e4 1495 /*
40d4e3df
ED		 1496 * You may not queue a frame bigger than the mtu. This is the lowest level
1497 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1498 */
1ce4f28b 1499
3bdc0eba
BG		 1500 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1501 if (!netif_supports_nofcs(dev)) {
1502 err = -EPROTONOSUPPORT;
1503 goto out_unlock;
1504 }
1505 extra_len = 4; /* We're doing our own CRC */
1506 }
1507
1da177e4 1508 err = -EMSGSIZE;
3bdc0eba 1509 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1510 goto out_unlock;
1511
1a35ca80
ED		 1512 if (!skb) {
1513 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1514 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1515 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1516
1517 rcu_read_unlock();
4ce40912 1518 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1519 if (skb == NULL)
1520 return -ENOBUFS;
1521 /* FIXME: Save some space for broken drivers that write a hard
1522 * header at transmission time by themselves. PPP is the notable
1523 * one here. This should really be fixed at the driver level.
1524 */
1525 skb_reserve(skb, reserved);
1526 skb_reset_network_header(skb);
1527
1528 /* Try to align data part correctly */
1529 if (hhlen) {
1530 skb->data -= hhlen;
1531 skb->tail -= hhlen;
1532 if (len < hhlen)
1533 skb_reset_network_header(skb);
1534 }
1535 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1536 if (err)
1537 goto out_free;
1538 goto retry;
1da177e4
LT		 1539 }
1540
3bdc0eba 1541 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1542 /* Earlier code assumed this would be a VLAN pkt,
1543 * double-check this now that we have the actual
1544 * packet in hand.
1545 */
1546 struct ethhdr *ehdr;
1547 skb_reset_mac_header(skb);
1548 ehdr = eth_hdr(skb);
1549 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1550 err = -EMSGSIZE;
1551 goto out_unlock;
1552 }
1553 }
1a35ca80 1554
1da177e4
LT		 1555 skb->protocol = proto;
1556 skb->dev = dev;
1557 skb->priority = sk->sk_priority;
2d37a186 1558 skb->mark = sk->sk_mark;
bf84a010
DB		 1559
1560 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1561
3bdc0eba
BG		 1562 if (unlikely(extra_len == 4))
1563 skb->no_fcs = 1;
1564
40893fd0 1565 skb_probe_transport_header(skb, 0);
c1aad275 1566
1da177e4 1567 dev_queue_xmit(skb);
654d1f8a 1568 rcu_read_unlock();
40d4e3df 1569 return len;
1da177e4 1570
1da177e4 1571out_unlock:
654d1f8a 1572 rcu_read_unlock();
1a35ca80
ED		 1573out_free:
1574 kfree_skb(skb);
1da177e4
LT		 1575 return err;
1576}
1da177e4 1577
eea49cc9 1578static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1579 const struct sock *sk,
dbcb5855 1580 unsigned int res)
1da177e4
LT		 1581{
1582 struct sk_filter *filter;
fda9ef5d 1583
80f8f102
ED		 1584 rcu_read_lock();
1585 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1586 if (filter != NULL)
0a14842f 1587 res = SK_RUN_FILTER(filter, skb);
80f8f102 1588 rcu_read_unlock();
1da177e4 1589
dbcb5855 1590 return res;
1da177e4
LT		 1591}
1592
1593/*
62ab0812
ED		 1594 * This function makes lazy skb cloning in hope that most of packets
1595 * are discarded by BPF.
1596 *
1597 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1598 * and skb->cb are mangled. It works because (and until) packets
1599 * falling here are owned by current CPU. Output packets are cloned
1600 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1601 * sequencially, so that if we return skb to original state on exit,
1602 * we will not harm anyone.
1da177e4
LT		 1603 */
1604
40d4e3df
ED		 1605static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1606 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1607{
1608 struct sock *sk;
1609 struct sockaddr_ll *sll;
1610 struct packet_sock *po;
40d4e3df 1611 u8 *skb_head = skb->data;
1da177e4 1612 int skb_len = skb->len;
dbcb5855 1613 unsigned int snaplen, res;
1da177e4
LT		 1614
1615 if (skb->pkt_type == PACKET_LOOPBACK)
1616 goto drop;
1617
1618 sk = pt->af_packet_priv;
1619 po = pkt_sk(sk);
1620
09ad9bc7 1621 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1622 goto drop;
1623
1da177e4
LT		 1624 skb->dev = dev;
1625
3b04ddde 1626 if (dev->header_ops) {
1da177e4 1627 /* The device has an explicit notion of ll header,
62ab0812
ED		 1628 * exported to higher levels.
1629 *
1630 * Otherwise, the device hides details of its frame
1631 * structure, so that corresponding packet head is
1632 * never delivered to user.
1da177e4
LT		 1633 */
1634 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1635 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1636 else if (skb->pkt_type == PACKET_OUTGOING) {
1637 /* Special case: outgoing packets have ll header at head */
bbe735e4 1638 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1639 }
1640 }
1641
1642 snaplen = skb->len;
1643
dbcb5855
DM		 1644 res = run_filter(skb, sk, snaplen);
1645 if (!res)
fda9ef5d 1646 goto drop_n_restore;
dbcb5855
DM		 1647 if (snaplen > res)
1648 snaplen = res;
1da177e4 1649
0fd7bac6 1650 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1651 goto drop_n_acct;
1652
1653 if (skb_shared(skb)) {
1654 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1655 if (nskb == NULL)
1656 goto drop_n_acct;
1657
1658 if (skb_head != skb->data) {
1659 skb->data = skb_head;
1660 skb->len = skb_len;
1661 }
abc4e4fa 1662 consume_skb(skb);
1da177e4
LT		 1663 skb = nskb;
1664 }
1665
ffbc6111
HX		 1666 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1667 sizeof(skb->cb));
1668
1669 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1670 sll->sll_family = AF_PACKET;
1671 sll->sll_hatype = dev->type;
1672 sll->sll_protocol = skb->protocol;
1673 sll->sll_pkttype = skb->pkt_type;
8032b464 1674 if (unlikely(po->origdev))
80feaacb
PWJ		 1675 sll->sll_ifindex = orig_dev->ifindex;
1676 else
1677 sll->sll_ifindex = dev->ifindex;
1da177e4 1678
b95cce35 1679 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1680
ffbc6111 1681 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1682
1da177e4
LT		 1683 if (pskb_trim(skb, snaplen))
1684 goto drop_n_acct;
1685
1686 skb_set_owner_r(skb, sk);
1687 skb->dev = NULL;
adf30907 1688 skb_dst_drop(skb);
1da177e4 1689
84531c24
PO		 1690 /* drop conntrack reference */
1691 nf_reset(skb);
1692
1da177e4 1693 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1694 po->stats.stats1.tp_packets++;
3b885787 1695 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1696 __skb_queue_tail(&sk->sk_receive_queue, skb);
1697 spin_unlock(&sk->sk_receive_queue.lock);
1698 sk->sk_data_ready(sk, skb->len);
1699 return 0;
1700
1701drop_n_acct:
7091fbd8 1702 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1703 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1704 atomic_inc(&sk->sk_drops);
1705 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1706
1707drop_n_restore:
1708 if (skb_head != skb->data && skb_shared(skb)) {
1709 skb->data = skb_head;
1710 skb->len = skb_len;
1711 }
1712drop:
ead2ceb0 1713 consume_skb(skb);
1da177e4
LT		 1714 return 0;
1715}
1716
40d4e3df
ED		 1717static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1718 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1719{
1720 struct sock *sk;
1721 struct packet_sock *po;
1722 struct sockaddr_ll *sll;
184f489e 1723 union tpacket_uhdr h;
40d4e3df 1724 u8 *skb_head = skb->data;
1da177e4 1725 int skb_len = skb->len;
dbcb5855 1726 unsigned int snaplen, res;
f6fb8f10 1727 unsigned long status = TP_STATUS_USER;
bbd6ef87 1728 unsigned short macoff, netoff, hdrlen;
1da177e4 1729 struct sk_buff *copy_skb = NULL;
bbd6ef87 1730 struct timespec ts;
b9c32fb2 1731 __u32 ts_status;
1da177e4
LT		 1732
1733 if (skb->pkt_type == PACKET_LOOPBACK)
1734 goto drop;
1735
1736 sk = pt->af_packet_priv;
1737 po = pkt_sk(sk);
1738
09ad9bc7 1739 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1740 goto drop;
1741
3b04ddde 1742 if (dev->header_ops) {
1da177e4 1743 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1744 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1745 else if (skb->pkt_type == PACKET_OUTGOING) {
1746 /* Special case: outgoing packets have ll header at head */
bbe735e4 1747 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1748 }
1749 }
1750
8dc41944
HX		 1751 if (skb->ip_summed == CHECKSUM_PARTIAL)
1752 status |= TP_STATUS_CSUMNOTREADY;
1753
1da177e4
LT		 1754 snaplen = skb->len;
1755
dbcb5855
DM		 1756 res = run_filter(skb, sk, snaplen);
1757 if (!res)
fda9ef5d 1758 goto drop_n_restore;
dbcb5855
DM		 1759 if (snaplen > res)
1760 snaplen = res;
1da177e4
LT		 1761
1762 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1763 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1764 po->tp_reserve;
1da177e4 1765 } else {
95c96174 1766 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1767 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1768 (maclen < 16 ? 16 : maclen)) +
1769 po->tp_reserve;
1da177e4
LT		 1770 macoff = netoff - maclen;
1771 }
f6fb8f10 1772 if (po->tp_version <= TPACKET_V2) {
1773 if (macoff + snaplen > po->rx_ring.frame_size) {
1774 if (po->copy_thresh &&
0fd7bac6 1775 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1776 if (skb_shared(skb)) {
1777 copy_skb = skb_clone(skb, GFP_ATOMIC);
1778 } else {
1779 copy_skb = skb_get(skb);
1780 skb_head = skb->data;
1781 }
1782 if (copy_skb)
1783 skb_set_owner_r(copy_skb, sk);
1da177e4 1784 }
f6fb8f10 1785 snaplen = po->rx_ring.frame_size - macoff;
1786 if ((int)snaplen < 0)
1787 snaplen = 0;
1da177e4 1788 }
1da177e4 1789 }
1da177e4 1790 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1791 h.raw = packet_current_rx_frame(po, skb,
1792 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1793 if (!h.raw)
1da177e4 1794 goto ring_is_full;
f6fb8f10 1795 if (po->tp_version <= TPACKET_V2) {
1796 packet_increment_rx_head(po, &po->rx_ring);
1797 /*
1798 * LOSING will be reported till you read the stats,
1799 * because it's COR - Clear On Read.
1800 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1801 * at packet level.
1802 */
ee80fbf3 1803 if (po->stats.stats1.tp_drops)
f6fb8f10 1804 status |= TP_STATUS_LOSING;
1805 }
ee80fbf3 1806 po->stats.stats1.tp_packets++;
1da177e4
LT		 1807 if (copy_skb) {
1808 status |= TP_STATUS_COPY;
1809 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1810 }
1da177e4
LT		 1811 spin_unlock(&sk->sk_receive_queue.lock);
1812
bbd6ef87 1813 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1814
1815 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1816 getnstimeofday(&ts);
1da177e4 1817
b9c32fb2
DB		 1818 status |= ts_status;
1819
bbd6ef87
PM		 1820 switch (po->tp_version) {
1821 case TPACKET_V1:
1822 h.h1->tp_len = skb->len;
1823 h.h1->tp_snaplen = snaplen;
1824 h.h1->tp_mac = macoff;
1825 h.h1->tp_net = netoff;
4b457bdf
DB		 1826 h.h1->tp_sec = ts.tv_sec;
1827 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1828 hdrlen = sizeof(*h.h1);
1829 break;
1830 case TPACKET_V2:
1831 h.h2->tp_len = skb->len;
1832 h.h2->tp_snaplen = snaplen;
1833 h.h2->tp_mac = macoff;
1834 h.h2->tp_net = netoff;
bbd6ef87
PM		 1835 h.h2->tp_sec = ts.tv_sec;
1836 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1837 if (vlan_tx_tag_present(skb)) {
1838 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1839 status |= TP_STATUS_VLAN_VALID;
1840 } else {
1841 h.h2->tp_vlan_tci = 0;
1842 }
13fcb7bd 1843 h.h2->tp_padding = 0;
bbd6ef87
PM		 1844 hdrlen = sizeof(*h.h2);
1845 break;
f6fb8f10 1846 case TPACKET_V3:
1847 /* tp_nxt_offset,vlan are already populated above.
1848 * So DONT clear those fields here
1849 */
1850 h.h3->tp_status |= status;
1851 h.h3->tp_len = skb->len;
1852 h.h3->tp_snaplen = snaplen;
1853 h.h3->tp_mac = macoff;
1854 h.h3->tp_net = netoff;
f6fb8f10 1855 h.h3->tp_sec = ts.tv_sec;
1856 h.h3->tp_nsec = ts.tv_nsec;
1857 hdrlen = sizeof(*h.h3);
1858 break;
bbd6ef87
PM		 1859 default:
1860 BUG();
1861 }
1da177e4 1862
bbd6ef87 1863 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1864 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1865 sll->sll_family = AF_PACKET;
1866 sll->sll_hatype = dev->type;
1867 sll->sll_protocol = skb->protocol;
1868 sll->sll_pkttype = skb->pkt_type;
8032b464 1869 if (unlikely(po->origdev))
80feaacb
PWJ		 1870 sll->sll_ifindex = orig_dev->ifindex;
1871 else
1872 sll->sll_ifindex = dev->ifindex;
1da177e4 1873
e16aa207 1874 smp_mb();
f6dafa95 1875#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1876 {
0af55bb5
CG		 1877 u8 *start, *end;
1878
f6fb8f10 1879 if (po->tp_version <= TPACKET_V2) {
1880 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1881 + macoff + snaplen);
1882 for (start = h.raw; start < end; start += PAGE_SIZE)
1883 flush_dcache_page(pgv_to_page(start));
1884 }
cc9f01b2 1885 smp_wmb();
1da177e4 1886 }
f6dafa95 1887#endif
f6fb8f10 1888 if (po->tp_version <= TPACKET_V2)
1889 __packet_set_status(po, h.raw, status);
1890 else
1891 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1892
1893 sk->sk_data_ready(sk, 0);
1894
1895drop_n_restore:
1896 if (skb_head != skb->data && skb_shared(skb)) {
1897 skb->data = skb_head;
1898 skb->len = skb_len;
1899 }
1900drop:
1ce4f28b 1901 kfree_skb(skb);
1da177e4
LT		 1902 return 0;
1903
1904ring_is_full:
ee80fbf3 1905 po->stats.stats1.tp_drops++;
1da177e4
LT		 1906 spin_unlock(&sk->sk_receive_queue.lock);
1907
1908 sk->sk_data_ready(sk, 0);
acb5d75b 1909 kfree_skb(copy_skb);
1da177e4
LT		 1910 goto drop_n_restore;
1911}
1912
69e3c75f
JB		 1913static void tpacket_destruct_skb(struct sk_buff *skb)
1914{
1915 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1916 void *ph;
1da177e4 1917
69e3c75f 1918 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 1919 __u32 ts;
1920
69e3c75f 1921 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1922 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1923 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 1924
1925 ts = __packet_set_timestamp(po, ph, skb);
1926 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 1927 }
1928
1929 sock_wfree(skb);
1930}
1931
40d4e3df
ED		 1932static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1933 void *frame, struct net_device *dev, int size_max,
ae641949 1934 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1935{
184f489e 1936 union tpacket_uhdr ph;
09effa67 1937 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 1938 struct socket *sock = po->sk.sk_socket;
1939 struct page *page;
1940 void *data;
1941 int err;
1942
1943 ph.raw = frame;
1944
1945 skb->protocol = proto;
1946 skb->dev = dev;
1947 skb->priority = po->sk.sk_priority;
2d37a186 1948 skb->mark = po->sk.sk_mark;
2e31396f 1949 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 1950 skb_shinfo(skb)->destructor_arg = ph.raw;
1951
1952 switch (po->tp_version) {
1953 case TPACKET_V2:
1954 tp_len = ph.h2->tp_len;
1955 break;
1956 default:
1957 tp_len = ph.h1->tp_len;
1958 break;
1959 }
09effa67
DM		 1960 if (unlikely(tp_len > size_max)) {
1961 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
1962 return -EMSGSIZE;
1963 }
69e3c75f 1964
ae641949 1965 skb_reserve(skb, hlen);
69e3c75f 1966 skb_reset_network_header(skb);
40893fd0 1967 skb_probe_transport_header(skb, 0);
c1aad275 1968
5920cd3a
PC		 1969 if (po->tp_tx_has_off) {
1970 int off_min, off_max, off;
1971 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1972 off_max = po->tx_ring.frame_size - tp_len;
1973 if (sock->type == SOCK_DGRAM) {
1974 switch (po->tp_version) {
1975 case TPACKET_V2:
1976 off = ph.h2->tp_net;
1977 break;
1978 default:
1979 off = ph.h1->tp_net;
1980 break;
1981 }
1982 } else {
1983 switch (po->tp_version) {
1984 case TPACKET_V2:
1985 off = ph.h2->tp_mac;
1986 break;
1987 default:
1988 off = ph.h1->tp_mac;
1989 break;
1990 }
1991 }
1992 if (unlikely((off < off_min) || (off_max < off)))
1993 return -EINVAL;
1994 data = ph.raw + off;
1995 } else {
1996 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1997 }
69e3c75f
JB		 1998 to_write = tp_len;
1999
2000 if (sock->type == SOCK_DGRAM) {
2001 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2002 NULL, tp_len);
2003 if (unlikely(err < 0))
2004 return -EINVAL;
40d4e3df 2005 } else if (dev->hard_header_len) {
69e3c75f
JB		 2006 /* net device doesn't like empty head */
2007 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2008 pr_err("packet size is too short (%d < %d)\n",
2009 tp_len, dev->hard_header_len);
69e3c75f
JB		 2010 return -EINVAL;
2011 }
2012
2013 skb_push(skb, dev->hard_header_len);
2014 err = skb_store_bits(skb, 0, data,
2015 dev->hard_header_len);
2016 if (unlikely(err))
2017 return err;
2018
2019 data += dev->hard_header_len;
2020 to_write -= dev->hard_header_len;
2021 }
2022
69e3c75f
JB		 2023 offset = offset_in_page(data);
2024 len_max = PAGE_SIZE - offset;
2025 len = ((to_write > len_max) ? len_max : to_write);
2026
2027 skb->data_len = to_write;
2028 skb->len += to_write;
2029 skb->truesize += to_write;
2030 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2031
2032 while (likely(to_write)) {
2033 nr_frags = skb_shinfo(skb)->nr_frags;
2034
2035 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2036 pr_err("Packet exceed the number of skb frags(%lu)\n",
2037 MAX_SKB_FRAGS);
69e3c75f
JB		 2038 return -EFAULT;
2039 }
2040
0af55bb5
CG		 2041 page = pgv_to_page(data);
2042 data += len;
69e3c75f
JB		 2043 flush_dcache_page(page);
2044 get_page(page);
0af55bb5 2045 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2046 to_write -= len;
2047 offset = 0;
2048 len_max = PAGE_SIZE;
2049 len = ((to_write > len_max) ? len_max : to_write);
2050 }
2051
2052 return tp_len;
2053}
2054
2055static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2056{
69e3c75f
JB		 2057 struct sk_buff *skb;
2058 struct net_device *dev;
2059 __be16 proto;
827d9780 2060 bool need_rls_dev = false;
09effa67 2061 int err, reserve = 0;
40d4e3df
ED		 2062 void *ph;
2063 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2064 int tp_len, size_max;
2065 unsigned char *addr;
2066 int len_sum = 0;
9e67030a 2067 int status = TP_STATUS_AVAILABLE;
ae641949 2068 int hlen, tlen;
69e3c75f 2069
69e3c75f
JB		 2070 mutex_lock(&po->pg_vec_lock);
2071
69e3c75f 2072 if (saddr == NULL) {
827d9780 2073 dev = po->prot_hook.dev;
69e3c75f
JB		 2074 proto = po->num;
2075 addr = NULL;
2076 } else {
2077 err = -EINVAL;
2078 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2079 goto out;
2080 if (msg->msg_namelen < (saddr->sll_halen
2081 + offsetof(struct sockaddr_ll,
2082 sll_addr)))
2083 goto out;
69e3c75f
JB		 2084 proto = saddr->sll_protocol;
2085 addr = saddr->sll_addr;
827d9780
BG		 2086 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2087 need_rls_dev = true;
69e3c75f
JB		 2088 }
2089
69e3c75f
JB		 2090 err = -ENXIO;
2091 if (unlikely(dev == NULL))
2092 goto out;
2093
09effa67
DM		 2094 reserve = dev->hard_header_len;
2095
69e3c75f
JB		 2096 err = -ENETDOWN;
2097 if (unlikely(!(dev->flags & IFF_UP)))
2098 goto out_put;
2099
2100 size_max = po->tx_ring.frame_size
b5dd884e 2101 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2102
09effa67
DM		 2103 if (size_max > dev->mtu + reserve)
2104 size_max = dev->mtu + reserve;
2105
69e3c75f
JB		 2106 do {
2107 ph = packet_current_frame(po, &po->tx_ring,
2108 TP_STATUS_SEND_REQUEST);
2109
2110 if (unlikely(ph == NULL)) {
2111 schedule();
2112 continue;
2113 }
2114
2115 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2116 hlen = LL_RESERVED_SPACE(dev);
2117 tlen = dev->needed_tailroom;
69e3c75f 2118 skb = sock_alloc_send_skb(&po->sk,
ae641949 2119 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2120 0, &err);
2121
2122 if (unlikely(skb == NULL))
2123 goto out_status;
2124
2125 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2126 addr, hlen);
69e3c75f
JB		 2127
2128 if (unlikely(tp_len < 0)) {
2129 if (po->tp_loss) {
2130 __packet_set_status(po, ph,
2131 TP_STATUS_AVAILABLE);
2132 packet_increment_head(&po->tx_ring);
2133 kfree_skb(skb);
2134 continue;
2135 } else {
2136 status = TP_STATUS_WRONG_FORMAT;
2137 err = tp_len;
2138 goto out_status;
2139 }
2140 }
2141
2142 skb->destructor = tpacket_destruct_skb;
2143 __packet_set_status(po, ph, TP_STATUS_SENDING);
2144 atomic_inc(&po->tx_ring.pending);
2145
2146 status = TP_STATUS_SEND_REQUEST;
2147 err = dev_queue_xmit(skb);
eb70df13
JP		 2148 if (unlikely(err > 0)) {
2149 err = net_xmit_errno(err);
2150 if (err && __packet_get_status(po, ph) ==
2151 TP_STATUS_AVAILABLE) {
2152 /* skb was destructed already */
2153 skb = NULL;
2154 goto out_status;
2155 }
2156 /*
2157 * skb was dropped but not destructed yet;
2158 * let's treat it like congestion or err < 0
2159 */
2160 err = 0;
2161 }
69e3c75f
JB		 2162 packet_increment_head(&po->tx_ring);
2163 len_sum += tp_len;
f64f9e71
JP		 2164 } while (likely((ph != NULL) ||
2165 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2166 (atomic_read(&po->tx_ring.pending))))
2167 );
69e3c75f
JB		 2168
2169 err = len_sum;
2170 goto out_put;
2171
69e3c75f
JB		 2172out_status:
2173 __packet_set_status(po, ph, status);
2174 kfree_skb(skb);
2175out_put:
827d9780
BG		 2176 if (need_rls_dev)
2177 dev_put(dev);
69e3c75f
JB		 2178out:
2179 mutex_unlock(&po->pg_vec_lock);
2180 return err;
2181}
69e3c75f 2182
eea49cc9
OJ		 2183static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2184 size_t reserve, size_t len,
2185 size_t linear, int noblock,
2186 int *err)
bfd5f4a3
SS		 2187{
2188 struct sk_buff *skb;
2189
2190 /* Under a page? Don't bother with paged skb. */
2191 if (prepad + len < PAGE_SIZE || !linear)
2192 linear = len;
2193
2194 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2195 err, 0);
bfd5f4a3
SS		 2196 if (!skb)
2197 return NULL;
2198
2199 skb_reserve(skb, reserve);
2200 skb_put(skb, linear);
2201 skb->data_len = len - linear;
2202 skb->len += len - linear;
2203
2204 return skb;
2205}
2206
69e3c75f 2207static int packet_snd(struct socket *sock,
1da177e4
LT		 2208 struct msghdr *msg, size_t len)
2209{
2210 struct sock *sk = sock->sk;
40d4e3df 2211 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2212 struct sk_buff *skb;
2213 struct net_device *dev;
0e11c91e 2214 __be16 proto;
827d9780 2215 bool need_rls_dev = false;
1da177e4 2216 unsigned char *addr;
827d9780 2217 int err, reserve = 0;
bfd5f4a3
SS		 2218 struct virtio_net_hdr vnet_hdr = { 0 };
2219 int offset = 0;
2220 int vnet_hdr_len;
2221 struct packet_sock *po = pkt_sk(sk);
2222 unsigned short gso_type = 0;
ae641949 2223 int hlen, tlen;
3bdc0eba 2224 int extra_len = 0;
1da177e4
LT		 2225
2226 /*
1ce4f28b 2227 * Get and verify the address.
1da177e4 2228 */
1ce4f28b 2229
1da177e4 2230 if (saddr == NULL) {
827d9780 2231 dev = po->prot_hook.dev;
1da177e4
LT		 2232 proto = po->num;
2233 addr = NULL;
2234 } else {
2235 err = -EINVAL;
2236 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2237 goto out;
0fb375fb
EB		 2238 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2239 goto out;
1da177e4
LT		 2240 proto = saddr->sll_protocol;
2241 addr = saddr->sll_addr;
827d9780
BG		 2242 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2243 need_rls_dev = true;
1da177e4
LT		 2244 }
2245
1da177e4
LT		 2246 err = -ENXIO;
2247 if (dev == NULL)
2248 goto out_unlock;
2249 if (sock->type == SOCK_RAW)
2250 reserve = dev->hard_header_len;
2251
d5e76b0a
DM		 2252 err = -ENETDOWN;
2253 if (!(dev->flags & IFF_UP))
2254 goto out_unlock;
2255
bfd5f4a3
SS		 2256 if (po->has_vnet_hdr) {
2257 vnet_hdr_len = sizeof(vnet_hdr);
2258
2259 err = -EINVAL;
2260 if (len < vnet_hdr_len)
2261 goto out_unlock;
2262
2263 len -= vnet_hdr_len;
2264
2265 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2266 vnet_hdr_len);
2267 if (err < 0)
2268 goto out_unlock;
2269
2270 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2271 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2272 vnet_hdr.hdr_len))
2273 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2274 vnet_hdr.csum_offset + 2;
2275
2276 err = -EINVAL;
2277 if (vnet_hdr.hdr_len > len)
2278 goto out_unlock;
2279
2280 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2281 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2282 case VIRTIO_NET_HDR_GSO_TCPV4:
2283 gso_type = SKB_GSO_TCPV4;
2284 break;
2285 case VIRTIO_NET_HDR_GSO_TCPV6:
2286 gso_type = SKB_GSO_TCPV6;
2287 break;
2288 case VIRTIO_NET_HDR_GSO_UDP:
2289 gso_type = SKB_GSO_UDP;
2290 break;
2291 default:
2292 goto out_unlock;
2293 }
2294
2295 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2296 gso_type |= SKB_GSO_TCP_ECN;
2297
2298 if (vnet_hdr.gso_size == 0)
2299 goto out_unlock;
2300
2301 }
2302 }
2303
3bdc0eba
BG		 2304 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2305 if (!netif_supports_nofcs(dev)) {
2306 err = -EPROTONOSUPPORT;
2307 goto out_unlock;
2308 }
2309 extra_len = 4; /* We're doing our own CRC */
2310 }
2311
1da177e4 2312 err = -EMSGSIZE;
3bdc0eba 2313 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2314 goto out_unlock;
2315
bfd5f4a3 2316 err = -ENOBUFS;
ae641949
HX		 2317 hlen = LL_RESERVED_SPACE(dev);
2318 tlen = dev->needed_tailroom;
2319 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2320 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2321 if (skb == NULL)
1da177e4
LT		 2322 goto out_unlock;
2323
bfd5f4a3 2324 skb_set_network_header(skb, reserve);
1da177e4 2325
0c4e8581
SH		 2326 err = -EINVAL;
2327 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2328 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2329 goto out_free;
1da177e4
LT		 2330
2331 /* Returns -EFAULT on error */
bfd5f4a3 2332 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2333 if (err)
2334 goto out_free;
bf84a010
DB		 2335
2336 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2337
3bdc0eba 2338 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2339 /* Earlier code assumed this would be a VLAN pkt,
2340 * double-check this now that we have the actual
2341 * packet in hand.
2342 */
2343 struct ethhdr *ehdr;
2344 skb_reset_mac_header(skb);
2345 ehdr = eth_hdr(skb);
2346 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2347 err = -EMSGSIZE;
2348 goto out_free;
2349 }
57f89bfa
BG		 2350 }
2351
09effa67
DM		 2352 skb->protocol = proto;
2353 skb->dev = dev;
1da177e4 2354 skb->priority = sk->sk_priority;
2d37a186 2355 skb->mark = sk->sk_mark;
1da177e4 2356
bfd5f4a3
SS		 2357 if (po->has_vnet_hdr) {
2358 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2359 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2360 vnet_hdr.csum_offset)) {
2361 err = -EINVAL;
2362 goto out_free;
2363 }
2364 }
2365
2366 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2367 skb_shinfo(skb)->gso_type = gso_type;
2368
2369 /* Header must be checked, and gso_segs computed. */
2370 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2371 skb_shinfo(skb)->gso_segs = 0;
2372
2373 len += vnet_hdr_len;
2374 }
2375
40893fd0 2376 skb_probe_transport_header(skb, reserve);
c1aad275 2377
3bdc0eba
BG		 2378 if (unlikely(extra_len == 4))
2379 skb->no_fcs = 1;
2380
1da177e4
LT		 2381 /*
2382 * Now send it
2383 */
2384
2385 err = dev_queue_xmit(skb);
2386 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2387 goto out_unlock;
2388
827d9780
BG		 2389 if (need_rls_dev)
2390 dev_put(dev);
1da177e4 2391
40d4e3df 2392 return len;
1da177e4
LT		 2393
2394out_free:
2395 kfree_skb(skb);
2396out_unlock:
827d9780 2397 if (dev && need_rls_dev)
1da177e4
LT		 2398 dev_put(dev);
2399out:
2400 return err;
2401}
2402
69e3c75f
JB		 2403static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2404 struct msghdr *msg, size_t len)
2405{
69e3c75f
JB		 2406 struct sock *sk = sock->sk;
2407 struct packet_sock *po = pkt_sk(sk);
2408 if (po->tx_ring.pg_vec)
2409 return tpacket_snd(po, msg);
2410 else
69e3c75f
JB		 2411 return packet_snd(sock, msg, len);
2412}
2413
1da177e4
LT		 2414/*
2415 * Close a PACKET socket. This is fairly simple. We immediately go
2416 * to 'closed' state and remove our protocol entry in the device list.
2417 */
2418
2419static int packet_release(struct socket *sock)
2420{
2421 struct sock *sk = sock->sk;
2422 struct packet_sock *po;
d12d01d6 2423 struct net *net;
f6fb8f10 2424 union tpacket_req_u req_u;
1da177e4
LT		 2425
2426 if (!sk)
2427 return 0;
2428
3b1e0a65 2429 net = sock_net(sk);
1da177e4
LT		 2430 po = pkt_sk(sk);
2431
0fa7fa98 2432 mutex_lock(&net->packet.sklist_lock);
808f5114 2433 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2434 mutex_unlock(&net->packet.sklist_lock);
2435
2436 preempt_disable();
920de804 2437 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2438 preempt_enable();
1da177e4 2439
808f5114 2440 spin_lock(&po->bind_lock);
ce06b03e 2441 unregister_prot_hook(sk, false);
160ff18a
BG		 2442 if (po->prot_hook.dev) {
2443 dev_put(po->prot_hook.dev);
2444 po->prot_hook.dev = NULL;
2445 }
808f5114 2446 spin_unlock(&po->bind_lock);
1da177e4 2447
1da177e4 2448 packet_flush_mclist(sk);
1da177e4 2449
9665d5d6
PS		 2450 if (po->rx_ring.pg_vec) {
2451 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2452 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2453 }
69e3c75f 2454
9665d5d6
PS		 2455 if (po->tx_ring.pg_vec) {
2456 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2457 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2458 }
1da177e4 2459
dc99f600
DM		 2460 fanout_release(sk);
2461
808f5114 2462 synchronize_net();
1da177e4
LT		 2463 /*
2464 * Now the socket is dead. No more input will appear.
2465 */
1da177e4
LT		 2466 sock_orphan(sk);
2467 sock->sk = NULL;
2468
2469 /* Purge queues */
2470
2471 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2472 sk_refcnt_debug_release(sk);
1da177e4
LT		 2473
2474 sock_put(sk);
2475 return 0;
2476}
2477
2478/*
2479 * Attach a packet hook.
2480 */
2481
0e11c91e 2482static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2483{
2484 struct packet_sock *po = pkt_sk(sk);
dc99f600 2485
aef950b4
WY		 2486 if (po->fanout) {
2487 if (dev)
2488 dev_put(dev);
2489
dc99f600 2490 return -EINVAL;
aef950b4 2491 }
1da177e4
LT		 2492
2493 lock_sock(sk);
2494
2495 spin_lock(&po->bind_lock);
ce06b03e 2496 unregister_prot_hook(sk, true);
1da177e4
LT		 2497 po->num = protocol;
2498 po->prot_hook.type = protocol;
160ff18a
BG		 2499 if (po->prot_hook.dev)
2500 dev_put(po->prot_hook.dev);
1da177e4
LT		 2501 po->prot_hook.dev = dev;
2502
2503 po->ifindex = dev ? dev->ifindex : 0;
2504
2505 if (protocol == 0)
2506 goto out_unlock;
2507
be85d4ad 2508 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2509 register_prot_hook(sk);
be85d4ad
UT		 2510 } else {
2511 sk->sk_err = ENETDOWN;
2512 if (!sock_flag(sk, SOCK_DEAD))
2513 sk->sk_error_report(sk);
1da177e4
LT		 2514 }
2515
2516out_unlock:
2517 spin_unlock(&po->bind_lock);
2518 release_sock(sk);
2519 return 0;
2520}
2521
2522/*
2523 * Bind a packet socket to a device
2524 */
2525
40d4e3df
ED		 2526static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2527 int addr_len)
1da177e4 2528{
40d4e3df 2529 struct sock *sk = sock->sk;
1da177e4
LT		 2530 char name[15];
2531 struct net_device *dev;
2532 int err = -ENODEV;
1ce4f28b 2533
1da177e4
LT		 2534 /*
2535 * Check legality
2536 */
1ce4f28b 2537
8ae55f04 2538 if (addr_len != sizeof(struct sockaddr))
1da177e4 2539 return -EINVAL;
40d4e3df 2540 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2541
3b1e0a65 2542 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2543 if (dev)
1da177e4 2544 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2545 return err;
2546}
1da177e4
LT		 2547
2548static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2549{
40d4e3df
ED		 2550 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2551 struct sock *sk = sock->sk;
1da177e4
LT		 2552 struct net_device *dev = NULL;
2553 int err;
2554
2555
2556 /*
2557 * Check legality
2558 */
1ce4f28b 2559
1da177e4
LT		 2560 if (addr_len < sizeof(struct sockaddr_ll))
2561 return -EINVAL;
2562 if (sll->sll_family != AF_PACKET)
2563 return -EINVAL;
2564
2565 if (sll->sll_ifindex) {
2566 err = -ENODEV;
3b1e0a65 2567 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2568 if (dev == NULL)
2569 goto out;
2570 }
2571 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2572
2573out:
2574 return err;
2575}
2576
2577static struct proto packet_proto = {
2578 .name = "PACKET",
2579 .owner = THIS_MODULE,
2580 .obj_size = sizeof(struct packet_sock),
2581};
2582
2583/*
1ce4f28b 2584 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2585 */
2586
3f378b68
EP		 2587static int packet_create(struct net *net, struct socket *sock, int protocol,
2588 int kern)
1da177e4
LT		 2589{
2590 struct sock *sk;
2591 struct packet_sock *po;
0e11c91e 2592 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2593 int err;
2594
df008c91 2595 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2596 return -EPERM;
be02097c
DM		 2597 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2598 sock->type != SOCK_PACKET)
1da177e4
LT		 2599 return -ESOCKTNOSUPPORT;
2600
2601 sock->state = SS_UNCONNECTED;
2602
2603 err = -ENOBUFS;
6257ff21 2604 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2605 if (sk == NULL)
2606 goto out;
2607
2608 sock->ops = &packet_ops;
1da177e4
LT		 2609 if (sock->type == SOCK_PACKET)
2610 sock->ops = &packet_ops_spkt;
be02097c 2611
1da177e4
LT		 2612 sock_init_data(sock, sk);
2613
2614 po = pkt_sk(sk);
2615 sk->sk_family = PF_PACKET;
0e11c91e 2616 po->num = proto;
1da177e4
LT		 2617
2618 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2619 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2620
2621 /*
2622 * Attach a protocol block
2623 */
2624
2625 spin_lock_init(&po->bind_lock);
905db440 2626 mutex_init(&po->pg_vec_lock);
1da177e4 2627 po->prot_hook.func = packet_rcv;
be02097c 2628
1da177e4
LT		 2629 if (sock->type == SOCK_PACKET)
2630 po->prot_hook.func = packet_rcv_spkt;
be02097c 2631
1da177e4
LT		 2632 po->prot_hook.af_packet_priv = sk;
2633
0e11c91e
AV		 2634 if (proto) {
2635 po->prot_hook.type = proto;
ce06b03e 2636 register_prot_hook(sk);
1da177e4
LT		 2637 }
2638
0fa7fa98 2639 mutex_lock(&net->packet.sklist_lock);
808f5114 2640 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2641 mutex_unlock(&net->packet.sklist_lock);
2642
2643 preempt_disable();
3680453c 2644 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2645 preempt_enable();
808f5114 2646
40d4e3df 2647 return 0;
1da177e4
LT		 2648out:
2649 return err;
2650}
2651
2652/*
2653 * Pull a packet from our receive queue and hand it to the user.
2654 * If necessary we block.
2655 */
2656
2657static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2658 struct msghdr *msg, size_t len, int flags)
2659{
2660 struct sock *sk = sock->sk;
2661 struct sk_buff *skb;
2662 int copied, err;
bfd5f4a3 2663 int vnet_hdr_len = 0;
1da177e4
LT		 2664
2665 err = -EINVAL;
ed85b565 2666 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2667 goto out;
2668
2669#if 0
2670 /* What error should we return now? EUNATTACH? */
2671 if (pkt_sk(sk)->ifindex < 0)
2672 return -ENODEV;
2673#endif
2674
ed85b565 2675 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2676 err = sock_recv_errqueue(sk, msg, len,
2677 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2678 goto out;
2679 }
2680
1da177e4
LT		 2681 /*
2682 * Call the generic datagram receiver. This handles all sorts
2683 * of horrible races and re-entrancy so we can forget about it
2684 * in the protocol layers.
2685 *
2686 * Now it will return ENETDOWN, if device have just gone down,
2687 * but then it will block.
2688 */
2689
40d4e3df 2690 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2691
2692 /*
1ce4f28b 2693 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2694 * handles the blocking we don't see and worry about blocking
2695 * retries.
2696 */
2697
8ae55f04 2698 if (skb == NULL)
1da177e4
LT		 2699 goto out;
2700
bfd5f4a3
SS		 2701 if (pkt_sk(sk)->has_vnet_hdr) {
2702 struct virtio_net_hdr vnet_hdr = { 0 };
2703
2704 err = -EINVAL;
2705 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2706 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2707 goto out_free;
2708
1f18b717
MK		 2709 len -= vnet_hdr_len;
2710
bfd5f4a3
SS		 2711 if (skb_is_gso(skb)) {
2712 struct skb_shared_info *sinfo = skb_shinfo(skb);
2713
2714 /* This is a hint as to how much should be linear. */
2715 vnet_hdr.hdr_len = skb_headlen(skb);
2716 vnet_hdr.gso_size = sinfo->gso_size;
2717 if (sinfo->gso_type & SKB_GSO_TCPV4)
2718 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2719 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2720 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2721 else if (sinfo->gso_type & SKB_GSO_UDP)
2722 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2723 else if (sinfo->gso_type & SKB_GSO_FCOE)
2724 goto out_free;
2725 else
2726 BUG();
2727 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2728 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2729 } else
2730 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2731
2732 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2733 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2734 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2735 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2736 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2737 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2738 } /* else everything is zero */
2739
2740 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2741 vnet_hdr_len);
2742 if (err < 0)
2743 goto out_free;
2744 }
2745
f3d33426
HFS		 2746 /* You lose any data beyond the buffer you gave. If it worries
2747 * a user program they can ask the device for its MTU
2748 * anyway.
1da177e4 2749 */
1da177e4 2750 copied = skb->len;
40d4e3df
ED		 2751 if (copied > len) {
2752 copied = len;
2753 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2754 }
2755
2756 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2757 if (err)
2758 goto out_free;
2759
3b885787 2760 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2761
f3d33426
HFS		 2762 if (msg->msg_name) {
2763 /* If the address length field is there to be filled
2764 * in, we fill it in now.
2765 */
2766 if (sock->type == SOCK_PACKET) {
2767 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2768 } else {
2769 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2770 msg->msg_namelen = sll->sll_halen +
2771 offsetof(struct sockaddr_ll, sll_addr);
2772 }
ffbc6111
HX		 2773 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2774 msg->msg_namelen);
f3d33426 2775 }
1da177e4 2776
8dc41944 2777 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2778 struct tpacket_auxdata aux;
2779
2780 aux.tp_status = TP_STATUS_USER;
2781 if (skb->ip_summed == CHECKSUM_PARTIAL)
2782 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2783 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2784 aux.tp_snaplen = skb->len;
2785 aux.tp_mac = 0;
bbe735e4 2786 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2787 if (vlan_tx_tag_present(skb)) {
2788 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2789 aux.tp_status |= TP_STATUS_VLAN_VALID;
2790 } else {
2791 aux.tp_vlan_tci = 0;
2792 }
13fcb7bd 2793 aux.tp_padding = 0;
ffbc6111 2794 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2795 }
2796
1da177e4
LT		 2797 /*
2798 * Free or return the buffer as appropriate. Again this
2799 * hides all the races and re-entrancy issues from us.
2800 */
bfd5f4a3 2801 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2802
2803out_free:
2804 skb_free_datagram(sk, skb);
2805out:
2806 return err;
2807}
2808
1da177e4
LT		 2809static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2810 int *uaddr_len, int peer)
2811{
2812 struct net_device *dev;
2813 struct sock *sk = sock->sk;
2814
2815 if (peer)
2816 return -EOPNOTSUPP;
2817
2818 uaddr->sa_family = AF_PACKET;
2dc85bf3 2819 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2820 rcu_read_lock();
2821 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2822 if (dev)
2dc85bf3 2823 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2824 rcu_read_unlock();
1da177e4
LT		 2825 *uaddr_len = sizeof(*uaddr);
2826
2827 return 0;
2828}
1da177e4
LT		 2829
2830static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2831 int *uaddr_len, int peer)
2832{
2833 struct net_device *dev;
2834 struct sock *sk = sock->sk;
2835 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2836 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2837
2838 if (peer)
2839 return -EOPNOTSUPP;
2840
2841 sll->sll_family = AF_PACKET;
2842 sll->sll_ifindex = po->ifindex;
2843 sll->sll_protocol = po->num;
67286640 2844 sll->sll_pkttype = 0;
654d1f8a
ED		 2845 rcu_read_lock();
2846 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2847 if (dev) {
2848 sll->sll_hatype = dev->type;
2849 sll->sll_halen = dev->addr_len;
2850 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2851 } else {
2852 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2853 sll->sll_halen = 0;
2854 }
654d1f8a 2855 rcu_read_unlock();
0fb375fb 2856 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2857
2858 return 0;
2859}
2860
2aeb0b88
WC		 2861static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2862 int what)
1da177e4
LT		 2863{
2864 switch (i->type) {
2865 case PACKET_MR_MULTICAST:
1162563f
JP		 2866 if (i->alen != dev->addr_len)
2867 return -EINVAL;
1da177e4 2868 if (what > 0)
22bedad3 2869 return dev_mc_add(dev, i->addr);
1da177e4 2870 else
22bedad3 2871 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2872 break;
2873 case PACKET_MR_PROMISC:
2aeb0b88 2874 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2875 break;
2876 case PACKET_MR_ALLMULTI:
2aeb0b88 2877 return dev_set_allmulti(dev, what);
1da177e4 2878 break;
d95ed927 2879 case PACKET_MR_UNICAST:
1162563f
JP		 2880 if (i->alen != dev->addr_len)
2881 return -EINVAL;
d95ed927 2882 if (what > 0)
a748ee24 2883 return dev_uc_add(dev, i->addr);
d95ed927 2884 else
a748ee24 2885 return dev_uc_del(dev, i->addr);
d95ed927 2886 break;
40d4e3df
ED		 2887 default:
2888 break;
1da177e4 2889 }
2aeb0b88 2890 return 0;
1da177e4
LT		 2891}
2892
2893static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2894{
40d4e3df 2895 for ( ; i; i = i->next) {
1da177e4
LT		 2896 if (i->ifindex == dev->ifindex)
2897 packet_dev_mc(dev, i, what);
2898 }
2899}
2900
0fb375fb 2901static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2902{
2903 struct packet_sock *po = pkt_sk(sk);
2904 struct packet_mclist *ml, *i;
2905 struct net_device *dev;
2906 int err;
2907
2908 rtnl_lock();
2909
2910 err = -ENODEV;
3b1e0a65 2911 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2912 if (!dev)
2913 goto done;
2914
2915 err = -EINVAL;
1162563f 2916 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2917 goto done;
2918
2919 err = -ENOBUFS;
8b3a7005 2920 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2921 if (i == NULL)
2922 goto done;
2923
2924 err = 0;
2925 for (ml = po->mclist; ml; ml = ml->next) {
2926 if (ml->ifindex == mreq->mr_ifindex &&
2927 ml->type == mreq->mr_type &&
2928 ml->alen == mreq->mr_alen &&
2929 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2930 ml->count++;
2931 /* Free the new element ... */
2932 kfree(i);
2933 goto done;
2934 }
2935 }
2936
2937 i->type = mreq->mr_type;
2938 i->ifindex = mreq->mr_ifindex;
2939 i->alen = mreq->mr_alen;
2940 memcpy(i->addr, mreq->mr_address, i->alen);
2941 i->count = 1;
2942 i->next = po->mclist;
2943 po->mclist = i;
2aeb0b88
WC		 2944 err = packet_dev_mc(dev, i, 1);
2945 if (err) {
2946 po->mclist = i->next;
2947 kfree(i);
2948 }
1da177e4
LT		 2949
2950done:
2951 rtnl_unlock();
2952 return err;
2953}
2954
0fb375fb 2955static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2956{
2957 struct packet_mclist *ml, **mlp;
2958
2959 rtnl_lock();
2960
2961 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2962 if (ml->ifindex == mreq->mr_ifindex &&
2963 ml->type == mreq->mr_type &&
2964 ml->alen == mreq->mr_alen &&
2965 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2966 if (--ml->count == 0) {
2967 struct net_device *dev;
2968 *mlp = ml->next;
ad959e76
ED		 2969 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2970 if (dev)
1da177e4 2971 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2972 kfree(ml);
2973 }
2974 rtnl_unlock();
2975 return 0;
2976 }
2977 }
2978 rtnl_unlock();
2979 return -EADDRNOTAVAIL;
2980}
2981
2982static void packet_flush_mclist(struct sock *sk)
2983{
2984 struct packet_sock *po = pkt_sk(sk);
2985 struct packet_mclist *ml;
2986
2987 if (!po->mclist)
2988 return;
2989
2990 rtnl_lock();
2991 while ((ml = po->mclist) != NULL) {
2992 struct net_device *dev;
2993
2994 po->mclist = ml->next;
ad959e76
ED		 2995 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2996 if (dev != NULL)
1da177e4 2997 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2998 kfree(ml);
2999 }
3000 rtnl_unlock();
3001}
1da177e4
LT		 3002
3003static int
b7058842 3004packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3005{
3006 struct sock *sk = sock->sk;
8dc41944 3007 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3008 int ret;
3009
3010 if (level != SOL_PACKET)
3011 return -ENOPROTOOPT;
3012
69e3c75f 3013 switch (optname) {
1ce4f28b 3014 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3015 case PACKET_DROP_MEMBERSHIP:
3016 {
0fb375fb
EB		 3017 struct packet_mreq_max mreq;
3018 int len = optlen;
3019 memset(&mreq, 0, sizeof(mreq));
3020 if (len < sizeof(struct packet_mreq))
1da177e4 3021 return -EINVAL;
0fb375fb
EB		 3022 if (len > sizeof(mreq))
3023 len = sizeof(mreq);
40d4e3df 3024 if (copy_from_user(&mreq, optval, len))
1da177e4 3025 return -EFAULT;
0fb375fb
EB		 3026 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3027 return -EINVAL;
1da177e4
LT		 3028 if (optname == PACKET_ADD_MEMBERSHIP)
3029 ret = packet_mc_add(sk, &mreq);
3030 else
3031 ret = packet_mc_drop(sk, &mreq);
3032 return ret;
3033 }
a2efcfa0 3034
1da177e4 3035 case PACKET_RX_RING:
69e3c75f 3036 case PACKET_TX_RING:
1da177e4 3037 {
f6fb8f10 3038 union tpacket_req_u req_u;
3039 int len;
1da177e4 3040
f6fb8f10 3041 switch (po->tp_version) {
3042 case TPACKET_V1:
3043 case TPACKET_V2:
3044 len = sizeof(req_u.req);
3045 break;
3046 case TPACKET_V3:
3047 default:
3048 len = sizeof(req_u.req3);
3049 break;
3050 }
3051 if (optlen < len)
1da177e4 3052 return -EINVAL;
bfd5f4a3
SS		 3053 if (pkt_sk(sk)->has_vnet_hdr)
3054 return -EINVAL;
f6fb8f10 3055 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3056 return -EFAULT;
f6fb8f10 3057 return packet_set_ring(sk, &req_u, 0,
3058 optname == PACKET_TX_RING);
1da177e4
LT		 3059 }
3060 case PACKET_COPY_THRESH:
3061 {
3062 int val;
3063
40d4e3df 3064 if (optlen != sizeof(val))
1da177e4 3065 return -EINVAL;
40d4e3df 3066 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3067 return -EFAULT;
3068
3069 pkt_sk(sk)->copy_thresh = val;
3070 return 0;
3071 }
bbd6ef87
PM		 3072 case PACKET_VERSION:
3073 {
3074 int val;
3075
3076 if (optlen != sizeof(val))
3077 return -EINVAL;
69e3c75f 3078 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3079 return -EBUSY;
3080 if (copy_from_user(&val, optval, sizeof(val)))
3081 return -EFAULT;
3082 switch (val) {
3083 case TPACKET_V1:
3084 case TPACKET_V2:
f6fb8f10 3085 case TPACKET_V3:
bbd6ef87
PM		 3086 po->tp_version = val;
3087 return 0;
3088 default:
3089 return -EINVAL;
3090 }
3091 }
8913336a
PM		 3092 case PACKET_RESERVE:
3093 {
3094 unsigned int val;
3095
3096 if (optlen != sizeof(val))
3097 return -EINVAL;
69e3c75f 3098 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3099 return -EBUSY;
3100 if (copy_from_user(&val, optval, sizeof(val)))
3101 return -EFAULT;
3102 po->tp_reserve = val;
3103 return 0;
3104 }
69e3c75f
JB		 3105 case PACKET_LOSS:
3106 {
3107 unsigned int val;
3108
3109 if (optlen != sizeof(val))
3110 return -EINVAL;
3111 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3112 return -EBUSY;
3113 if (copy_from_user(&val, optval, sizeof(val)))
3114 return -EFAULT;
3115 po->tp_loss = !!val;
3116 return 0;
3117 }
8dc41944
HX		 3118 case PACKET_AUXDATA:
3119 {
3120 int val;
3121
3122 if (optlen < sizeof(val))
3123 return -EINVAL;
3124 if (copy_from_user(&val, optval, sizeof(val)))
3125 return -EFAULT;
3126
3127 po->auxdata = !!val;
3128 return 0;
3129 }
80feaacb
PWJ		 3130 case PACKET_ORIGDEV:
3131 {
3132 int val;
3133
3134 if (optlen < sizeof(val))
3135 return -EINVAL;
3136 if (copy_from_user(&val, optval, sizeof(val)))
3137 return -EFAULT;
3138
3139 po->origdev = !!val;
3140 return 0;
3141 }
bfd5f4a3
SS		 3142 case PACKET_VNET_HDR:
3143 {
3144 int val;
3145
3146 if (sock->type != SOCK_RAW)
3147 return -EINVAL;
3148 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3149 return -EBUSY;
3150 if (optlen < sizeof(val))
3151 return -EINVAL;
3152 if (copy_from_user(&val, optval, sizeof(val)))
3153 return -EFAULT;
3154
3155 po->has_vnet_hdr = !!val;
3156 return 0;
3157 }
614f60fa
SM		 3158 case PACKET_TIMESTAMP:
3159 {
3160 int val;
3161
3162 if (optlen != sizeof(val))
3163 return -EINVAL;
3164 if (copy_from_user(&val, optval, sizeof(val)))
3165 return -EFAULT;
3166
3167 po->tp_tstamp = val;
3168 return 0;
3169 }
dc99f600
DM		 3170 case PACKET_FANOUT:
3171 {
3172 int val;
3173
3174 if (optlen != sizeof(val))
3175 return -EINVAL;
3176 if (copy_from_user(&val, optval, sizeof(val)))
3177 return -EFAULT;
3178
3179 return fanout_add(sk, val & 0xffff, val >> 16);
3180 }
5920cd3a
PC		 3181 case PACKET_TX_HAS_OFF:
3182 {
3183 unsigned int val;
3184
3185 if (optlen != sizeof(val))
3186 return -EINVAL;
3187 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3188 return -EBUSY;
3189 if (copy_from_user(&val, optval, sizeof(val)))
3190 return -EFAULT;
3191 po->tp_tx_has_off = !!val;
3192 return 0;
3193 }
1da177e4
LT		 3194 default:
3195 return -ENOPROTOOPT;
3196 }
3197}
3198
3199static int packet_getsockopt(struct socket *sock, int level, int optname,
3200 char __user *optval, int __user *optlen)
3201{
3202 int len;
c06fff6e 3203 int val, lv = sizeof(val);
1da177e4
LT		 3204 struct sock *sk = sock->sk;
3205 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3206 void *data = &val;
ee80fbf3 3207 union tpacket_stats_u st;
1da177e4
LT		 3208
3209 if (level != SOL_PACKET)
3210 return -ENOPROTOOPT;
3211
8ae55f04
KK		 3212 if (get_user(len, optlen))
3213 return -EFAULT;
1da177e4
LT		 3214
3215 if (len < 0)
3216 return -EINVAL;
1ce4f28b 3217
69e3c75f 3218 switch (optname) {
1da177e4 3219 case PACKET_STATISTICS:
1da177e4 3220 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3221 memcpy(&st, &po->stats, sizeof(st));
3222 memset(&po->stats, 0, sizeof(po->stats));
3223 spin_unlock_bh(&sk->sk_receive_queue.lock);
3224
f6fb8f10 3225 if (po->tp_version == TPACKET_V3) {
c06fff6e 3226 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3227 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3228 data = &st.stats3;
f6fb8f10 3229 } else {
c06fff6e 3230 lv = sizeof(struct tpacket_stats);
8bcdeaff 3231 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3232 data = &st.stats1;
f6fb8f10 3233 }
ee80fbf3 3234
8dc41944
HX		 3235 break;
3236 case PACKET_AUXDATA:
8dc41944 3237 val = po->auxdata;
80feaacb
PWJ		 3238 break;
3239 case PACKET_ORIGDEV:
80feaacb 3240 val = po->origdev;
bfd5f4a3
SS		 3241 break;
3242 case PACKET_VNET_HDR:
bfd5f4a3 3243 val = po->has_vnet_hdr;
1da177e4 3244 break;
bbd6ef87 3245 case PACKET_VERSION:
bbd6ef87 3246 val = po->tp_version;
bbd6ef87
PM		 3247 break;
3248 case PACKET_HDRLEN:
3249 if (len > sizeof(int))
3250 len = sizeof(int);
3251 if (copy_from_user(&val, optval, len))
3252 return -EFAULT;
3253 switch (val) {
3254 case TPACKET_V1:
3255 val = sizeof(struct tpacket_hdr);
3256 break;
3257 case TPACKET_V2:
3258 val = sizeof(struct tpacket2_hdr);
3259 break;
f6fb8f10 3260 case TPACKET_V3:
3261 val = sizeof(struct tpacket3_hdr);
3262 break;
bbd6ef87
PM		 3263 default:
3264 return -EINVAL;
3265 }
bbd6ef87 3266 break;
8913336a 3267 case PACKET_RESERVE:
8913336a 3268 val = po->tp_reserve;
8913336a 3269 break;
69e3c75f 3270 case PACKET_LOSS:
69e3c75f 3271 val = po->tp_loss;
69e3c75f 3272 break;
614f60fa 3273 case PACKET_TIMESTAMP:
614f60fa 3274 val = po->tp_tstamp;
614f60fa 3275 break;
dc99f600 3276 case PACKET_FANOUT:
dc99f600
DM		 3277 val = (po->fanout ?
3278 ((u32)po->fanout->id |
77f65ebd
WB		 3279 ((u32)po->fanout->type << 16) |
3280 ((u32)po->fanout->flags << 24)) :
dc99f600 3281 0);
dc99f600 3282 break;
5920cd3a
PC		 3283 case PACKET_TX_HAS_OFF:
3284 val = po->tp_tx_has_off;
3285 break;
1da177e4
LT		 3286 default:
3287 return -ENOPROTOOPT;
3288 }
3289
c06fff6e
ED		 3290 if (len > lv)
3291 len = lv;
8ae55f04
KK		 3292 if (put_user(len, optlen))
3293 return -EFAULT;
8dc41944
HX		 3294 if (copy_to_user(optval, data, len))
3295 return -EFAULT;
8ae55f04 3296 return 0;
1da177e4
LT		 3297}
3298
3299
351638e7
JP		 3300static int packet_notifier(struct notifier_block *this,
3301 unsigned long msg, void *ptr)
1da177e4
LT		 3302{
3303 struct sock *sk;
351638e7 3304 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3305 struct net *net = dev_net(dev);
1da177e4 3306
808f5114 3307 rcu_read_lock();
b67bfe0d 3308 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3309 struct packet_sock *po = pkt_sk(sk);
3310
3311 switch (msg) {
3312 case NETDEV_UNREGISTER:
1da177e4
LT		 3313 if (po->mclist)
3314 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3315 /* fallthrough */
3316
1da177e4
LT		 3317 case NETDEV_DOWN:
3318 if (dev->ifindex == po->ifindex) {
3319 spin_lock(&po->bind_lock);
3320 if (po->running) {
ce06b03e 3321 __unregister_prot_hook(sk, false);
1da177e4
LT		 3322 sk->sk_err = ENETDOWN;
3323 if (!sock_flag(sk, SOCK_DEAD))
3324 sk->sk_error_report(sk);
3325 }
3326 if (msg == NETDEV_UNREGISTER) {
3327 po->ifindex = -1;
160ff18a
BG		 3328 if (po->prot_hook.dev)
3329 dev_put(po->prot_hook.dev);
1da177e4
LT		 3330 po->prot_hook.dev = NULL;
3331 }
3332 spin_unlock(&po->bind_lock);
3333 }
3334 break;
3335 case NETDEV_UP:
808f5114 3336 if (dev->ifindex == po->ifindex) {
3337 spin_lock(&po->bind_lock);
ce06b03e
DM		 3338 if (po->num)
3339 register_prot_hook(sk);
808f5114 3340 spin_unlock(&po->bind_lock);
1da177e4 3341 }
1da177e4
LT		 3342 break;
3343 }
3344 }
808f5114 3345 rcu_read_unlock();
1da177e4
LT		 3346 return NOTIFY_DONE;
3347}
3348
3349
3350static int packet_ioctl(struct socket *sock, unsigned int cmd,
3351 unsigned long arg)
3352{
3353 struct sock *sk = sock->sk;
3354
69e3c75f 3355 switch (cmd) {
40d4e3df
ED		 3356 case SIOCOUTQ:
3357 {
3358 int amount = sk_wmem_alloc_get(sk);
31e6d363 3359
40d4e3df
ED		 3360 return put_user(amount, (int __user *)arg);
3361 }
3362 case SIOCINQ:
3363 {
3364 struct sk_buff *skb;
3365 int amount = 0;
3366
3367 spin_lock_bh(&sk->sk_receive_queue.lock);
3368 skb = skb_peek(&sk->sk_receive_queue);
3369 if (skb)
3370 amount = skb->len;
3371 spin_unlock_bh(&sk->sk_receive_queue.lock);
3372 return put_user(amount, (int __user *)arg);
3373 }
3374 case SIOCGSTAMP:
3375 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3376 case SIOCGSTAMPNS:
3377 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3378
1da177e4 3379#ifdef CONFIG_INET
40d4e3df
ED		 3380 case SIOCADDRT:
3381 case SIOCDELRT:
3382 case SIOCDARP:
3383 case SIOCGARP:
3384 case SIOCSARP:
3385 case SIOCGIFADDR:
3386 case SIOCSIFADDR:
3387 case SIOCGIFBRDADDR:
3388 case SIOCSIFBRDADDR:
3389 case SIOCGIFNETMASK:
3390 case SIOCSIFNETMASK:
3391 case SIOCGIFDSTADDR:
3392 case SIOCSIFDSTADDR:
3393 case SIOCSIFFLAGS:
40d4e3df 3394 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3395#endif
3396
40d4e3df
ED		 3397 default:
3398 return -ENOIOCTLCMD;
1da177e4
LT		 3399 }
3400 return 0;
3401}
3402
40d4e3df 3403static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3404 poll_table *wait)
3405{
3406 struct sock *sk = sock->sk;
3407 struct packet_sock *po = pkt_sk(sk);
3408 unsigned int mask = datagram_poll(file, sock, wait);
3409
3410 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3411 if (po->rx_ring.pg_vec) {
f6fb8f10 3412 if (!packet_previous_rx_frame(po, &po->rx_ring,
3413 TP_STATUS_KERNEL))
1da177e4
LT		 3414 mask |= POLLIN | POLLRDNORM;
3415 }
3416 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3417 spin_lock_bh(&sk->sk_write_queue.lock);
3418 if (po->tx_ring.pg_vec) {
3419 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3420 mask |= POLLOUT | POLLWRNORM;
3421 }
3422 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3423 return mask;
3424}
3425
3426
3427/* Dirty? Well, I still did not learn better way to account
3428 * for user mmaps.
3429 */
3430
3431static void packet_mm_open(struct vm_area_struct *vma)
3432{
3433 struct file *file = vma->vm_file;
40d4e3df 3434 struct socket *sock = file->private_data;
1da177e4 3435 struct sock *sk = sock->sk;
1ce4f28b 3436
1da177e4
LT		 3437 if (sk)
3438 atomic_inc(&pkt_sk(sk)->mapped);
3439}
3440
3441static void packet_mm_close(struct vm_area_struct *vma)
3442{
3443 struct file *file = vma->vm_file;
40d4e3df 3444 struct socket *sock = file->private_data;
1da177e4 3445 struct sock *sk = sock->sk;
1ce4f28b 3446
1da177e4
LT		 3447 if (sk)
3448 atomic_dec(&pkt_sk(sk)->mapped);
3449}
3450
f0f37e2f 3451static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3452 .open = packet_mm_open,
3453 .close = packet_mm_close,
1da177e4
LT		 3454};
3455
0e3125c7
NH		 3456static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3457 unsigned int len)
1da177e4
LT		 3458{
3459 int i;
3460
4ebf0ae2 3461 for (i = 0; i < len; i++) {
0e3125c7 3462 if (likely(pg_vec[i].buffer)) {
c56b4d90 3463 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3464 vfree(pg_vec[i].buffer);
3465 else
3466 free_pages((unsigned long)pg_vec[i].buffer,
3467 order);
3468 pg_vec[i].buffer = NULL;
3469 }
1da177e4
LT		 3470 }
3471 kfree(pg_vec);
3472}
3473
eea49cc9 3474static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3475{
0e3125c7
NH		 3476 char *buffer = NULL;
3477 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3478 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3479
3480 buffer = (char *) __get_free_pages(gfp_flags, order);
3481
3482 if (buffer)
3483 return buffer;
3484
3485 /*
3486 * __get_free_pages failed, fall back to vmalloc
3487 */
bbce5a59 3488 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3489
0e3125c7
NH		 3490 if (buffer)
3491 return buffer;
3492
3493 /*
3494 * vmalloc failed, lets dig into swap here
3495 */
0e3125c7
NH		 3496 gfp_flags &= ~__GFP_NORETRY;
3497 buffer = (char *)__get_free_pages(gfp_flags, order);
3498 if (buffer)
3499 return buffer;
3500
3501 /*
3502 * complete and utter failure
3503 */
3504 return NULL;
4ebf0ae2
DM		 3505}
3506
0e3125c7 3507static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3508{
3509 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3510 struct pgv *pg_vec;
4ebf0ae2
DM		 3511 int i;
3512
0e3125c7 3513 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3514 if (unlikely(!pg_vec))
3515 goto out;
3516
3517 for (i = 0; i < block_nr; i++) {
c56b4d90 3518 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3519 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3520 goto out_free_pgvec;
3521 }
3522
3523out:
3524 return pg_vec;
3525
3526out_free_pgvec:
3527 free_pg_vec(pg_vec, order, block_nr);
3528 pg_vec = NULL;
3529 goto out;
3530}
1da177e4 3531
f6fb8f10 3532static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3533 int closing, int tx_ring)
1da177e4 3534{
0e3125c7 3535 struct pgv *pg_vec = NULL;
1da177e4 3536 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3537 int was_running, order = 0;
69e3c75f
JB		 3538 struct packet_ring_buffer *rb;
3539 struct sk_buff_head *rb_queue;
0e11c91e 3540 __be16 num;
f6fb8f10 3541 int err = -EINVAL;
3542 /* Added to avoid minimal code churn */
3543 struct tpacket_req *req = &req_u->req;
3544
3545 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3546 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3547 WARN(1, "Tx-ring is not supported.\n");
3548 goto out;
3549 }
1ce4f28b 3550
69e3c75f
JB		 3551 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3552 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3553
69e3c75f
JB		 3554 err = -EBUSY;
3555 if (!closing) {
3556 if (atomic_read(&po->mapped))
3557 goto out;
3558 if (atomic_read(&rb->pending))
3559 goto out;
3560 }
1da177e4 3561
69e3c75f
JB		 3562 if (req->tp_block_nr) {
3563 /* Sanity tests and some calculations */
3564 err = -EBUSY;
3565 if (unlikely(rb->pg_vec))
3566 goto out;
1da177e4 3567
bbd6ef87
PM		 3568 switch (po->tp_version) {
3569 case TPACKET_V1:
3570 po->tp_hdrlen = TPACKET_HDRLEN;
3571 break;
3572 case TPACKET_V2:
3573 po->tp_hdrlen = TPACKET2_HDRLEN;
3574 break;
f6fb8f10 3575 case TPACKET_V3:
3576 po->tp_hdrlen = TPACKET3_HDRLEN;
3577 break;
bbd6ef87
PM		 3578 }
3579
69e3c75f 3580 err = -EINVAL;
4ebf0ae2 3581 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3582 goto out;
4ebf0ae2 3583 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3584 goto out;
8913336a 3585 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3586 po->tp_reserve))
3587 goto out;
4ebf0ae2 3588 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3589 goto out;
1da177e4 3590
69e3c75f
JB		 3591 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3592 if (unlikely(rb->frames_per_block <= 0))
3593 goto out;
3594 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3595 req->tp_frame_nr))
3596 goto out;
1da177e4
LT		 3597
3598 err = -ENOMEM;
4ebf0ae2
DM		 3599 order = get_order(req->tp_block_size);
3600 pg_vec = alloc_pg_vec(req, order);
3601 if (unlikely(!pg_vec))
1da177e4 3602 goto out;
f6fb8f10 3603 switch (po->tp_version) {
3604 case TPACKET_V3:
3605 /* Transmit path is not supported. We checked
3606 * it above but just being paranoid
3607 */
3608 if (!tx_ring)
3609 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3610 break;
3611 default:
3612 break;
3613 }
69e3c75f
JB		 3614 }
3615 /* Done */
3616 else {
3617 err = -EINVAL;
4ebf0ae2 3618 if (unlikely(req->tp_frame_nr))
69e3c75f 3619 goto out;
1da177e4
LT		 3620 }
3621
3622 lock_sock(sk);
3623
3624 /* Detach socket from network */
3625 spin_lock(&po->bind_lock);
3626 was_running = po->running;
3627 num = po->num;
3628 if (was_running) {
1da177e4 3629 po->num = 0;
ce06b03e 3630 __unregister_prot_hook(sk, false);
1da177e4
LT		 3631 }
3632 spin_unlock(&po->bind_lock);
1ce4f28b 3633
1da177e4
LT		 3634 synchronize_net();
3635
3636 err = -EBUSY;
905db440 3637 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3638 if (closing || atomic_read(&po->mapped) == 0) {
3639 err = 0;
69e3c75f 3640 spin_lock_bh(&rb_queue->lock);
c053fd96 3641 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3642 rb->frame_max = (req->tp_frame_nr - 1);
3643 rb->head = 0;
3644 rb->frame_size = req->tp_frame_size;
3645 spin_unlock_bh(&rb_queue->lock);
3646
c053fd96
CG		 3647 swap(rb->pg_vec_order, order);
3648 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3649
3650 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3651 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3652 tpacket_rcv : packet_rcv;
3653 skb_queue_purge(rb_queue);
1da177e4 3654 if (atomic_read(&po->mapped))
40d4e3df
ED		 3655 pr_err("packet_mmap: vma is busy: %d\n",
3656 atomic_read(&po->mapped));
1da177e4 3657 }
905db440 3658 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3659
3660 spin_lock(&po->bind_lock);
ce06b03e 3661 if (was_running) {
1da177e4 3662 po->num = num;
ce06b03e 3663 register_prot_hook(sk);
1da177e4
LT		 3664 }
3665 spin_unlock(&po->bind_lock);
f6fb8f10 3666 if (closing && (po->tp_version > TPACKET_V2)) {
3667 /* Because we don't support block-based V3 on tx-ring */
3668 if (!tx_ring)
3669 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3670 }
1da177e4
LT		 3671 release_sock(sk);
3672
1da177e4
LT		 3673 if (pg_vec)
3674 free_pg_vec(pg_vec, order, req->tp_block_nr);
3675out:
3676 return err;
3677}
3678
69e3c75f
JB		 3679static int packet_mmap(struct file *file, struct socket *sock,
3680 struct vm_area_struct *vma)
1da177e4
LT		 3681{
3682 struct sock *sk = sock->sk;
3683 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3684 unsigned long size, expected_size;
3685 struct packet_ring_buffer *rb;
1da177e4
LT		 3686 unsigned long start;
3687 int err = -EINVAL;
3688 int i;
3689
3690 if (vma->vm_pgoff)
3691 return -EINVAL;
3692
905db440 3693 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3694
3695 expected_size = 0;
3696 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3697 if (rb->pg_vec) {
3698 expected_size += rb->pg_vec_len
3699 * rb->pg_vec_pages
3700 * PAGE_SIZE;
3701 }
3702 }
3703
3704 if (expected_size == 0)
1da177e4 3705 goto out;
69e3c75f
JB		 3706
3707 size = vma->vm_end - vma->vm_start;
3708 if (size != expected_size)
1da177e4
LT		 3709 goto out;
3710
1da177e4 3711 start = vma->vm_start;
69e3c75f
JB		 3712 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3713 if (rb->pg_vec == NULL)
3714 continue;
3715
3716 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3717 struct page *page;
3718 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3719 int pg_num;
3720
c56b4d90
CG		 3721 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3722 page = pgv_to_page(kaddr);
69e3c75f
JB		 3723 err = vm_insert_page(vma, start, page);
3724 if (unlikely(err))
3725 goto out;
3726 start += PAGE_SIZE;
0e3125c7 3727 kaddr += PAGE_SIZE;
69e3c75f 3728 }
4ebf0ae2 3729 }
1da177e4 3730 }
69e3c75f 3731
4ebf0ae2 3732 atomic_inc(&po->mapped);
1da177e4
LT		 3733 vma->vm_ops = &packet_mmap_ops;
3734 err = 0;
3735
3736out:
905db440 3737 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3738 return err;
3739}
1da177e4 3740
90ddc4f0 3741static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3742 .family = PF_PACKET,
3743 .owner = THIS_MODULE,
3744 .release = packet_release,
3745 .bind = packet_bind_spkt,
3746 .connect = sock_no_connect,
3747 .socketpair = sock_no_socketpair,
3748 .accept = sock_no_accept,
3749 .getname = packet_getname_spkt,
3750 .poll = datagram_poll,
3751 .ioctl = packet_ioctl,
3752 .listen = sock_no_listen,
3753 .shutdown = sock_no_shutdown,
3754 .setsockopt = sock_no_setsockopt,
3755 .getsockopt = sock_no_getsockopt,
3756 .sendmsg = packet_sendmsg_spkt,
3757 .recvmsg = packet_recvmsg,
3758 .mmap = sock_no_mmap,
3759 .sendpage = sock_no_sendpage,
3760};
1da177e4 3761
90ddc4f0 3762static const struct proto_ops packet_ops = {
1da177e4
LT		 3763 .family = PF_PACKET,
3764 .owner = THIS_MODULE,
3765 .release = packet_release,
3766 .bind = packet_bind,
3767 .connect = sock_no_connect,
3768 .socketpair = sock_no_socketpair,
3769 .accept = sock_no_accept,
1ce4f28b 3770 .getname = packet_getname,
1da177e4
LT		 3771 .poll = packet_poll,
3772 .ioctl = packet_ioctl,
3773 .listen = sock_no_listen,
3774 .shutdown = sock_no_shutdown,
3775 .setsockopt = packet_setsockopt,
3776 .getsockopt = packet_getsockopt,
3777 .sendmsg = packet_sendmsg,
3778 .recvmsg = packet_recvmsg,
3779 .mmap = packet_mmap,
3780 .sendpage = sock_no_sendpage,
3781};
3782
ec1b4cf7 3783static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3784 .family = PF_PACKET,
3785 .create = packet_create,
3786 .owner = THIS_MODULE,
3787};
3788
3789static struct notifier_block packet_netdev_notifier = {
40d4e3df 3790 .notifier_call = packet_notifier,
1da177e4
LT		 3791};
3792
3793#ifdef CONFIG_PROC_FS
1da177e4
LT		 3794
3795static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3796 __acquires(RCU)
1da177e4 3797{
e372c414 3798 struct net *net = seq_file_net(seq);
808f5114 3799
3800 rcu_read_lock();
3801 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3802}
3803
3804static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3805{
1bf40954 3806 struct net *net = seq_file_net(seq);
808f5114 3807 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3808}
3809
3810static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3811 __releases(RCU)
1da177e4 3812{
808f5114 3813 rcu_read_unlock();
1da177e4
LT		 3814}
3815
1ce4f28b 3816static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3817{
3818 if (v == SEQ_START_TOKEN)
3819 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3820 else {
b7ceabd9 3821 struct sock *s = sk_entry(v);
1da177e4
LT		 3822 const struct packet_sock *po = pkt_sk(s);
3823
3824 seq_printf(seq,
71338aa7 3825 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3826 s,
3827 atomic_read(&s->sk_refcnt),
3828 s->sk_type,
3829 ntohs(po->num),
3830 po->ifindex,
3831 po->running,
3832 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3833 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3834 sock_i_ino(s));
1da177e4
LT		 3835 }
3836
3837 return 0;
3838}
3839
56b3d975 3840static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3841 .start = packet_seq_start,
3842 .next = packet_seq_next,
3843 .stop = packet_seq_stop,
3844 .show = packet_seq_show,
3845};
3846
3847static int packet_seq_open(struct inode *inode, struct file *file)
3848{
e372c414
DL		 3849 return seq_open_net(inode, file, &packet_seq_ops,
3850 sizeof(struct seq_net_private));
1da177e4
LT		 3851}
3852
da7071d7 3853static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3854 .owner = THIS_MODULE,
3855 .open = packet_seq_open,
3856 .read = seq_read,
3857 .llseek = seq_lseek,
e372c414 3858 .release = seq_release_net,
1da177e4
LT		 3859};
3860
3861#endif
3862
2c8c1e72 3863static int __net_init packet_net_init(struct net *net)
d12d01d6 3864{
0fa7fa98 3865 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3866 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3867
d4beaa66 3868 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3869 return -ENOMEM;
3870
3871 return 0;
3872}
3873
2c8c1e72 3874static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3875{
ece31ffd 3876 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3877}
3878
3879static struct pernet_operations packet_net_ops = {
3880 .init = packet_net_init,
3881 .exit = packet_net_exit,
3882};
3883
3884
1da177e4
LT		 3885static void __exit packet_exit(void)
3886{
1da177e4 3887 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3888 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3889 sock_unregister(PF_PACKET);
3890 proto_unregister(&packet_proto);
3891}
3892
3893static int __init packet_init(void)
3894{
3895 int rc = proto_register(&packet_proto, 0);
3896
3897 if (rc != 0)
3898 goto out;
3899
3900 sock_register(&packet_family_ops);
d12d01d6 3901 register_pernet_subsys(&packet_net_ops);
1da177e4 3902 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3903out:
3904 return rc;
3905}
3906
3907module_init(packet_init);
3908module_exit(packet_exit);
3909MODULE_LICENSE("GPL");
3910MODULE_ALIAS_NETPROTO(PF_PACKET);
ubuntu-zesty-kernel mirror