]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - net/packet/af_packet.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'for-v3.13-fixes' of git://git.infradead.org/battery-2.6
[mirror_ubuntu-jammy-kernel.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
5df0ddfb 91#include <linux/reciprocal_div.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
66e56cd4
DB		 240static struct net_device *packet_cached_dev_get(struct packet_sock *po)
241{
242 struct net_device *dev;
243
244 rcu_read_lock();
245 dev = rcu_dereference(po->cached_dev);
246 if (likely(dev))
247 dev_hold(dev);
248 rcu_read_unlock();
249
250 return dev;
251}
252
253static void packet_cached_dev_assign(struct packet_sock *po,
254 struct net_device *dev)
255{
256 rcu_assign_pointer(po->cached_dev, dev);
257}
258
259static void packet_cached_dev_reset(struct packet_sock *po)
260{
261 RCU_INIT_POINTER(po->cached_dev, NULL);
262}
263
ce06b03e
DM		 264/* register_prot_hook must be invoked with the po->bind_lock held,
265 * or from a context in which asynchronous accesses to the packet
266 * socket is not possible (packet_create()).
267 */
268static void register_prot_hook(struct sock *sk)
269{
270 struct packet_sock *po = pkt_sk(sk);
e40526cb 271
ce06b03e 272 if (!po->running) {
66e56cd4 273 if (po->fanout)
dc99f600 274 __fanout_link(sk, po);
66e56cd4 275 else
dc99f600 276 dev_add_pack(&po->prot_hook);
e40526cb 277
ce06b03e
DM		 278 sock_hold(sk);
279 po->running = 1;
280 }
281}
282
283/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
284 * held. If the sync parameter is true, we will temporarily drop
285 * the po->bind_lock and do a synchronize_net to make sure no
286 * asynchronous packet processing paths still refer to the elements
287 * of po->prot_hook. If the sync parameter is false, it is the
288 * callers responsibility to take care of this.
289 */
290static void __unregister_prot_hook(struct sock *sk, bool sync)
291{
292 struct packet_sock *po = pkt_sk(sk);
293
294 po->running = 0;
66e56cd4
DB		 295
296 if (po->fanout)
dc99f600 297 __fanout_unlink(sk, po);
66e56cd4 298 else
dc99f600 299 __dev_remove_pack(&po->prot_hook);
e40526cb 300
ce06b03e
DM		 301 __sock_put(sk);
302
303 if (sync) {
304 spin_unlock(&po->bind_lock);
305 synchronize_net();
306 spin_lock(&po->bind_lock);
307 }
308}
309
310static void unregister_prot_hook(struct sock *sk, bool sync)
311{
312 struct packet_sock *po = pkt_sk(sk);
313
314 if (po->running)
315 __unregister_prot_hook(sk, sync);
316}
317
f6dafa95 318static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 319{
320 if (is_vmalloc_addr(addr))
321 return vmalloc_to_page(addr);
322 return virt_to_page(addr);
323}
324
69e3c75f 325static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 326{
184f489e 327 union tpacket_uhdr h;
1da177e4 328
69e3c75f 329 h.raw = frame;
bbd6ef87
PM		 330 switch (po->tp_version) {
331 case TPACKET_V1:
69e3c75f 332 h.h1->tp_status = status;
0af55bb5 333 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 334 break;
335 case TPACKET_V2:
69e3c75f 336 h.h2->tp_status = status;
0af55bb5 337 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 338 break;
f6fb8f10 339 case TPACKET_V3:
69e3c75f 340 default:
f6fb8f10 341 WARN(1, "TPACKET version not supported.\n");
69e3c75f 342 BUG();
bbd6ef87 343 }
69e3c75f
JB		 344
345 smp_wmb();
bbd6ef87
PM		 346}
347
69e3c75f 348static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 349{
184f489e 350 union tpacket_uhdr h;
bbd6ef87 351
69e3c75f
JB		 352 smp_rmb();
353
bbd6ef87
PM		 354 h.raw = frame;
355 switch (po->tp_version) {
356 case TPACKET_V1:
0af55bb5 357 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 358 return h.h1->tp_status;
bbd6ef87 359 case TPACKET_V2:
0af55bb5 360 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 361 return h.h2->tp_status;
f6fb8f10 362 case TPACKET_V3:
69e3c75f 363 default:
f6fb8f10 364 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 365 BUG();
366 return 0;
bbd6ef87 367 }
1da177e4 368}
69e3c75f 369
b9c32fb2
DB		 370static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
371 unsigned int flags)
7a51384c
DB		 372{
373 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
374
375 if (shhwtstamps) {
376 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
377 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 378 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 379 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
380 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 381 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 382 }
383
384 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 385 return TP_STATUS_TS_SOFTWARE;
7a51384c 386
b9c32fb2 387 return 0;
7a51384c
DB		 388}
389
b9c32fb2
DB		 390static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
391 struct sk_buff *skb)
2e31396f
WB		 392{
393 union tpacket_uhdr h;
394 struct timespec ts;
b9c32fb2 395 __u32 ts_status;
2e31396f 396
b9c32fb2
DB		 397 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
398 return 0;
2e31396f
WB		 399
400 h.raw = frame;
401 switch (po->tp_version) {
402 case TPACKET_V1:
403 h.h1->tp_sec = ts.tv_sec;
404 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
405 break;
406 case TPACKET_V2:
407 h.h2->tp_sec = ts.tv_sec;
408 h.h2->tp_nsec = ts.tv_nsec;
409 break;
410 case TPACKET_V3:
411 default:
412 WARN(1, "TPACKET version not supported.\n");
413 BUG();
414 }
415
416 /* one flush is safe, as both fields always lie on the same cacheline */
417 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
418 smp_wmb();
b9c32fb2
DB		 419
420 return ts_status;
2e31396f
WB		 421}
422
69e3c75f
JB		 423static void *packet_lookup_frame(struct packet_sock *po,
424 struct packet_ring_buffer *rb,
425 unsigned int position,
426 int status)
427{
428 unsigned int pg_vec_pos, frame_offset;
184f489e 429 union tpacket_uhdr h;
69e3c75f
JB		 430
431 pg_vec_pos = position / rb->frames_per_block;
432 frame_offset = position % rb->frames_per_block;
433
0e3125c7
NH		 434 h.raw = rb->pg_vec[pg_vec_pos].buffer +
435 (frame_offset * rb->frame_size);
69e3c75f
JB		 436
437 if (status != __packet_get_status(po, h.raw))
438 return NULL;
439
440 return h.raw;
441}
442
eea49cc9 443static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 444 struct packet_ring_buffer *rb,
445 int status)
446{
447 return packet_lookup_frame(po, rb, rb->head, status);
448}
449
bc59ba39 450static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 451{
452 del_timer_sync(&pkc->retire_blk_timer);
453}
454
455static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
456 int tx_ring,
457 struct sk_buff_head *rb_queue)
458{
bc59ba39 459 struct tpacket_kbdq_core *pkc;
f6fb8f10 460
461 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
462
ec6f809f 463 spin_lock_bh(&rb_queue->lock);
f6fb8f10 464 pkc->delete_blk_timer = 1;
ec6f809f 465 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 466
467 prb_del_retire_blk_timer(pkc);
468}
469
470static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 471 struct tpacket_kbdq_core *pkc,
f6fb8f10 472 void (*func) (unsigned long))
473{
474 init_timer(&pkc->retire_blk_timer);
475 pkc->retire_blk_timer.data = (long)po;
476 pkc->retire_blk_timer.function = func;
477 pkc->retire_blk_timer.expires = jiffies;
478}
479
480static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
481{
bc59ba39 482 struct tpacket_kbdq_core *pkc;
f6fb8f10 483
484 if (tx_ring)
485 BUG();
486
487 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
488 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
489}
490
491static int prb_calc_retire_blk_tmo(struct packet_sock *po,
492 int blk_size_in_bytes)
493{
494 struct net_device *dev;
495 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 496 struct ethtool_cmd ecmd;
497 int err;
e440cf2c 498 u32 speed;
f6fb8f10 499
4bc71cb9
JP		 500 rtnl_lock();
501 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
502 if (unlikely(!dev)) {
503 rtnl_unlock();
f6fb8f10 504 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 505 }
506 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 507 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 508 rtnl_unlock();
509 if (!err) {
4bc71cb9
JP		 510 /*
511 * If the link speed is so slow you don't really
512 * need to worry about perf anyways
513 */
e440cf2c 514 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 515 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 516 } else {
517 msec = 1;
518 div = speed / 1000;
f6fb8f10 519 }
520 }
521
522 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
523
524 if (div)
525 mbits /= div;
526
527 tmo = mbits * msec;
528
529 if (div)
530 return tmo+1;
531 return tmo;
532}
533
bc59ba39 534static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 535 union tpacket_req_u *req_u)
536{
537 p1->feature_req_word = req_u->req3.tp_feature_req_word;
538}
539
540static void init_prb_bdqc(struct packet_sock *po,
541 struct packet_ring_buffer *rb,
542 struct pgv *pg_vec,
543 union tpacket_req_u *req_u, int tx_ring)
544{
bc59ba39 545 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
546 struct tpacket_block_desc *pbd;
f6fb8f10 547
548 memset(p1, 0x0, sizeof(*p1));
549
550 p1->knxt_seq_num = 1;
551 p1->pkbdq = pg_vec;
bc59ba39 552 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 553 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 554 p1->kblk_size = req_u->req3.tp_block_size;
555 p1->knum_blocks = req_u->req3.tp_block_nr;
556 p1->hdrlen = po->tp_hdrlen;
557 p1->version = po->tp_version;
558 p1->last_kactive_blk_num = 0;
ee80fbf3 559 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 560 if (req_u->req3.tp_retire_blk_tov)
561 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
562 else
563 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
564 req_u->req3.tp_block_size);
565 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
566 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
567
568 prb_init_ft_ops(p1, req_u);
569 prb_setup_retire_blk_timer(po, tx_ring);
570 prb_open_block(p1, pbd);
571}
572
573/* Do NOT update the last_blk_num first.
574 * Assumes sk_buff_head lock is held.
575 */
bc59ba39 576static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 577{
578 mod_timer(&pkc->retire_blk_timer,
579 jiffies + pkc->tov_in_jiffies);
580 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
581}
582
583/*
584 * Timer logic:
585 * 1) We refresh the timer only when we open a block.
586 * By doing this we don't waste cycles refreshing the timer
587 * on packet-by-packet basis.
588 *
589 * With a 1MB block-size, on a 1Gbps line, it will take
590 * i) ~8 ms to fill a block + ii) memcpy etc.
591 * In this cut we are not accounting for the memcpy time.
592 *
593 * So, if the user sets the 'tmo' to 10ms then the timer
594 * will never fire while the block is still getting filled
595 * (which is what we want). However, the user could choose
596 * to close a block early and that's fine.
597 *
598 * But when the timer does fire, we check whether or not to refresh it.
599 * Since the tmo granularity is in msecs, it is not too expensive
600 * to refresh the timer, lets say every '8' msecs.
601 * Either the user can set the 'tmo' or we can derive it based on
602 * a) line-speed and b) block-size.
603 * prb_calc_retire_blk_tmo() calculates the tmo.
604 *
605 */
606static void prb_retire_rx_blk_timer_expired(unsigned long data)
607{
608 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 609 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 610 unsigned int frozen;
bc59ba39 611 struct tpacket_block_desc *pbd;
f6fb8f10 612
613 spin_lock(&po->sk.sk_receive_queue.lock);
614
615 frozen = prb_queue_frozen(pkc);
616 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
617
618 if (unlikely(pkc->delete_blk_timer))
619 goto out;
620
621 /* We only need to plug the race when the block is partially filled.
622 * tpacket_rcv:
623 * lock(); increment BLOCK_NUM_PKTS; unlock()
624 * copy_bits() is in progress ...
625 * timer fires on other cpu:
626 * we can't retire the current block because copy_bits
627 * is in progress.
628 *
629 */
630 if (BLOCK_NUM_PKTS(pbd)) {
631 while (atomic_read(&pkc->blk_fill_in_prog)) {
632 /* Waiting for skb_copy_bits to finish... */
633 cpu_relax();
634 }
635 }
636
637 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
638 if (!frozen) {
639 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
640 if (!prb_dispatch_next_block(pkc, po))
641 goto refresh_timer;
642 else
643 goto out;
644 } else {
645 /* Case 1. Queue was frozen because user-space was
646 * lagging behind.
647 */
648 if (prb_curr_blk_in_use(pkc, pbd)) {
649 /*
650 * Ok, user-space is still behind.
651 * So just refresh the timer.
652 */
653 goto refresh_timer;
654 } else {
655 /* Case 2. queue was frozen,user-space caught up,
656 * now the link went idle && the timer fired.
657 * We don't have a block to close.So we open this
658 * block and restart the timer.
659 * opening a block thaws the queue,restarts timer
660 * Thawing/timer-refresh is a side effect.
661 */
662 prb_open_block(pkc, pbd);
663 goto out;
664 }
665 }
666 }
667
668refresh_timer:
669 _prb_refresh_rx_retire_blk_timer(pkc);
670
671out:
672 spin_unlock(&po->sk.sk_receive_queue.lock);
673}
674
eea49cc9 675static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 676 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 677{
678 /* Flush everything minus the block header */
679
680#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
681 u8 *start, *end;
682
683 start = (u8 *)pbd1;
684
685 /* Skip the block header(we know header WILL fit in 4K) */
686 start += PAGE_SIZE;
687
688 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
689 for (; start < end; start += PAGE_SIZE)
690 flush_dcache_page(pgv_to_page(start));
691
692 smp_wmb();
693#endif
694
695 /* Now update the block status. */
696
697 BLOCK_STATUS(pbd1) = status;
698
699 /* Flush the block header */
700
701#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
702 start = (u8 *)pbd1;
703 flush_dcache_page(pgv_to_page(start));
704
705 smp_wmb();
706#endif
707}
708
709/*
710 * Side effect:
711 *
712 * 1) flush the block
713 * 2) Increment active_blk_num
714 *
715 * Note:We DONT refresh the timer on purpose.
716 * Because almost always the next block will be opened.
717 */
bc59ba39 718static void prb_close_block(struct tpacket_kbdq_core *pkc1,
719 struct tpacket_block_desc *pbd1,
f6fb8f10 720 struct packet_sock *po, unsigned int stat)
721{
722 __u32 status = TP_STATUS_USER | stat;
723
724 struct tpacket3_hdr *last_pkt;
bc59ba39 725 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 726
ee80fbf3 727 if (po->stats.stats3.tp_drops)
f6fb8f10 728 status |= TP_STATUS_LOSING;
729
730 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
731 last_pkt->tp_next_offset = 0;
732
733 /* Get the ts of the last pkt */
734 if (BLOCK_NUM_PKTS(pbd1)) {
735 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
736 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
737 } else {
738 /* Ok, we tmo'd - so get the current time */
739 struct timespec ts;
740 getnstimeofday(&ts);
741 h1->ts_last_pkt.ts_sec = ts.tv_sec;
742 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
743 }
744
745 smp_wmb();
746
747 /* Flush the block */
748 prb_flush_block(pkc1, pbd1, status);
749
750 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
751}
752
eea49cc9 753static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 754{
755 pkc->reset_pending_on_curr_blk = 0;
756}
757
758/*
759 * Side effect of opening a block:
760 *
761 * 1) prb_queue is thawed.
762 * 2) retire_blk_timer is refreshed.
763 *
764 */
bc59ba39 765static void prb_open_block(struct tpacket_kbdq_core *pkc1,
766 struct tpacket_block_desc *pbd1)
f6fb8f10 767{
768 struct timespec ts;
bc59ba39 769 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 770
771 smp_rmb();
772
8da3056c
DB		 773 /* We could have just memset this but we will lose the
774 * flexibility of making the priv area sticky
775 */
f6fb8f10 776
8da3056c
DB		 777 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
778 BLOCK_NUM_PKTS(pbd1) = 0;
779 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 780
8da3056c
DB		 781 getnstimeofday(&ts);
782
783 h1->ts_first_pkt.ts_sec = ts.tv_sec;
784 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 785
8da3056c
DB		 786 pkc1->pkblk_start = (char *)pbd1;
787 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
788
789 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
790 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
791
792 pbd1->version = pkc1->version;
793 pkc1->prev = pkc1->nxt_offset;
794 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
795
796 prb_thaw_queue(pkc1);
797 _prb_refresh_rx_retire_blk_timer(pkc1);
798
799 smp_wmb();
f6fb8f10 800}
801
802/*
803 * Queue freeze logic:
804 * 1) Assume tp_block_nr = 8 blocks.
805 * 2) At time 't0', user opens Rx ring.
806 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
807 * 4) user-space is either sleeping or processing block '0'.
808 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
809 * it will close block-7,loop around and try to fill block '0'.
810 * call-flow:
811 * __packet_lookup_frame_in_block
812 * prb_retire_current_block()
813 * prb_dispatch_next_block()
814 * |->(BLOCK_STATUS == USER) evaluates to true
815 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
816 * 6) Now there are two cases:
817 * 6.1) Link goes idle right after the queue is frozen.
818 * But remember, the last open_block() refreshed the timer.
819 * When this timer expires,it will refresh itself so that we can
820 * re-open block-0 in near future.
821 * 6.2) Link is busy and keeps on receiving packets. This is a simple
822 * case and __packet_lookup_frame_in_block will check if block-0
823 * is free and can now be re-used.
824 */
eea49cc9 825static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 826 struct packet_sock *po)
827{
828 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 829 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 830}
831
832#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
833
834/*
835 * If the next block is free then we will dispatch it
836 * and return a good offset.
837 * Else, we will freeze the queue.
838 * So, caller must check the return value.
839 */
bc59ba39 840static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 841 struct packet_sock *po)
842{
bc59ba39 843 struct tpacket_block_desc *pbd;
f6fb8f10 844
845 smp_rmb();
846
847 /* 1. Get current block num */
848 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
849
850 /* 2. If this block is currently in_use then freeze the queue */
851 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
852 prb_freeze_queue(pkc, po);
853 return NULL;
854 }
855
856 /*
857 * 3.
858 * open this block and return the offset where the first packet
859 * needs to get stored.
860 */
861 prb_open_block(pkc, pbd);
862 return (void *)pkc->nxt_offset;
863}
864
bc59ba39 865static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 866 struct packet_sock *po, unsigned int status)
867{
bc59ba39 868 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 869
870 /* retire/close the current block */
871 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
872 /*
873 * Plug the case where copy_bits() is in progress on
874 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
875 * have space to copy the pkt in the current block and
876 * called prb_retire_current_block()
877 *
878 * We don't need to worry about the TMO case because
879 * the timer-handler already handled this case.
880 */
881 if (!(status & TP_STATUS_BLK_TMO)) {
882 while (atomic_read(&pkc->blk_fill_in_prog)) {
883 /* Waiting for skb_copy_bits to finish... */
884 cpu_relax();
885 }
886 }
887 prb_close_block(pkc, pbd, po, status);
888 return;
889 }
f6fb8f10 890}
891
eea49cc9 892static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 893 struct tpacket_block_desc *pbd)
f6fb8f10 894{
895 return TP_STATUS_USER & BLOCK_STATUS(pbd);
896}
897
eea49cc9 898static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 899{
900 return pkc->reset_pending_on_curr_blk;
901}
902
eea49cc9 903static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 904{
bc59ba39 905 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 906 atomic_dec(&pkc->blk_fill_in_prog);
907}
908
eea49cc9 909static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 910 struct tpacket3_hdr *ppd)
911{
912 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
913}
914
eea49cc9 915static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 916 struct tpacket3_hdr *ppd)
917{
918 ppd->hv1.tp_rxhash = 0;
919}
920
eea49cc9 921static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 922 struct tpacket3_hdr *ppd)
923{
924 if (vlan_tx_tag_present(pkc->skb)) {
925 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
926 ppd->tp_status = TP_STATUS_VLAN_VALID;
927 } else {
9e67030a
DB		 928 ppd->hv1.tp_vlan_tci = 0;
929 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 930 }
931}
932
bc59ba39 933static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 934 struct tpacket3_hdr *ppd)
935{
936 prb_fill_vlan_info(pkc, ppd);
937
938 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
939 prb_fill_rxhash(pkc, ppd);
940 else
941 prb_clear_rxhash(pkc, ppd);
942}
943
eea49cc9 944static void prb_fill_curr_block(char *curr,
bc59ba39 945 struct tpacket_kbdq_core *pkc,
946 struct tpacket_block_desc *pbd,
f6fb8f10 947 unsigned int len)
948{
949 struct tpacket3_hdr *ppd;
950
951 ppd = (struct tpacket3_hdr *)curr;
952 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
953 pkc->prev = curr;
954 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
955 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
956 BLOCK_NUM_PKTS(pbd) += 1;
957 atomic_inc(&pkc->blk_fill_in_prog);
958 prb_run_all_ft_ops(pkc, ppd);
959}
960
961/* Assumes caller has the sk->rx_queue.lock */
962static void *__packet_lookup_frame_in_block(struct packet_sock *po,
963 struct sk_buff *skb,
964 int status,
965 unsigned int len
966 )
967{
bc59ba39 968 struct tpacket_kbdq_core *pkc;
969 struct tpacket_block_desc *pbd;
f6fb8f10 970 char *curr, *end;
971
e3192690 972 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 973 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
974
975 /* Queue is frozen when user space is lagging behind */
976 if (prb_queue_frozen(pkc)) {
977 /*
978 * Check if that last block which caused the queue to freeze,
979 * is still in_use by user-space.
980 */
981 if (prb_curr_blk_in_use(pkc, pbd)) {
982 /* Can't record this packet */
983 return NULL;
984 } else {
985 /*
986 * Ok, the block was released by user-space.
987 * Now let's open that block.
988 * opening a block also thaws the queue.
989 * Thawing is a side effect.
990 */
991 prb_open_block(pkc, pbd);
992 }
993 }
994
995 smp_mb();
996 curr = pkc->nxt_offset;
997 pkc->skb = skb;
e3192690 998 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 999
1000 /* first try the current block */
1001 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1002 prb_fill_curr_block(curr, pkc, pbd, len);
1003 return (void *)curr;
1004 }
1005
1006 /* Ok, close the current block */
1007 prb_retire_current_block(pkc, po, 0);
1008
1009 /* Now, try to dispatch the next block */
1010 curr = (char *)prb_dispatch_next_block(pkc, po);
1011 if (curr) {
1012 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1013 prb_fill_curr_block(curr, pkc, pbd, len);
1014 return (void *)curr;
1015 }
1016
1017 /*
1018 * No free blocks are available.user_space hasn't caught up yet.
1019 * Queue was just frozen and now this packet will get dropped.
1020 */
1021 return NULL;
1022}
1023
eea49cc9 1024static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1025 struct sk_buff *skb,
1026 int status, unsigned int len)
1027{
1028 char *curr = NULL;
1029 switch (po->tp_version) {
1030 case TPACKET_V1:
1031 case TPACKET_V2:
1032 curr = packet_lookup_frame(po, &po->rx_ring,
1033 po->rx_ring.head, status);
1034 return curr;
1035 case TPACKET_V3:
1036 return __packet_lookup_frame_in_block(po, skb, status, len);
1037 default:
1038 WARN(1, "TPACKET version not supported\n");
1039 BUG();
99aa3473 1040 return NULL;
f6fb8f10 1041 }
1042}
1043
eea49cc9 1044static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1045 struct packet_ring_buffer *rb,
77f65ebd 1046 unsigned int idx,
f6fb8f10 1047 int status)
1048{
bc59ba39 1049 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1050 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1051
1052 if (status != BLOCK_STATUS(pbd))
1053 return NULL;
1054 return pbd;
1055}
1056
eea49cc9 1057static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1058{
1059 unsigned int prev;
1060 if (rb->prb_bdqc.kactive_blk_num)
1061 prev = rb->prb_bdqc.kactive_blk_num-1;
1062 else
1063 prev = rb->prb_bdqc.knum_blocks-1;
1064 return prev;
1065}
1066
1067/* Assumes caller has held the rx_queue.lock */
eea49cc9 1068static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1069 struct packet_ring_buffer *rb,
1070 int status)
1071{
1072 unsigned int previous = prb_previous_blk_num(rb);
1073 return prb_lookup_block(po, rb, previous, status);
1074}
1075
eea49cc9 1076static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1077 struct packet_ring_buffer *rb,
1078 int status)
1079{
1080 if (po->tp_version <= TPACKET_V2)
1081 return packet_previous_frame(po, rb, status);
1082
1083 return __prb_previous_block(po, rb, status);
1084}
1085
eea49cc9 1086static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1087 struct packet_ring_buffer *rb)
1088{
1089 switch (po->tp_version) {
1090 case TPACKET_V1:
1091 case TPACKET_V2:
1092 return packet_increment_head(rb);
1093 case TPACKET_V3:
1094 default:
1095 WARN(1, "TPACKET version not supported.\n");
1096 BUG();
1097 return;
1098 }
1099}
1100
eea49cc9 1101static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1102 struct packet_ring_buffer *rb,
1103 int status)
1104{
1105 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1106 return packet_lookup_frame(po, rb, previous, status);
1107}
1108
eea49cc9 1109static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1110{
1111 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1112}
1113
77f65ebd
WB		 1114static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1115{
1116 struct sock *sk = &po->sk;
1117 bool has_room;
1118
1119 if (po->prot_hook.func != tpacket_rcv)
1120 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1121 <= sk->sk_rcvbuf;
1122
1123 spin_lock(&sk->sk_receive_queue.lock);
1124 if (po->tp_version == TPACKET_V3)
1125 has_room = prb_lookup_block(po, &po->rx_ring,
1126 po->rx_ring.prb_bdqc.kactive_blk_num,
1127 TP_STATUS_KERNEL);
1128 else
1129 has_room = packet_lookup_frame(po, &po->rx_ring,
1130 po->rx_ring.head,
1131 TP_STATUS_KERNEL);
1132 spin_unlock(&sk->sk_receive_queue.lock);
1133
1134 return has_room;
1135}
1136
1da177e4
LT		 1137static void packet_sock_destruct(struct sock *sk)
1138{
ed85b565
RC		 1139 skb_queue_purge(&sk->sk_error_queue);
1140
547b792c
IJ		 1141 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1142 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1143
1144 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1145 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1146 return;
1147 }
1148
17ab56a2 1149 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1150}
1151
dc99f600
DM		 1152static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1153{
1154 int x = atomic_read(&f->rr_cur) + 1;
1155
1156 if (x >= num)
1157 x = 0;
1158
1159 return x;
1160}
1161
77f65ebd
WB		 1162static unsigned int fanout_demux_hash(struct packet_fanout *f,
1163 struct sk_buff *skb,
1164 unsigned int num)
dc99f600 1165{
f55d112e 1166 return reciprocal_divide(skb->rxhash, num);
dc99f600
DM		 1167}
1168
77f65ebd
WB		 1169static unsigned int fanout_demux_lb(struct packet_fanout *f,
1170 struct sk_buff *skb,
1171 unsigned int num)
dc99f600
DM		 1172{
1173 int cur, old;
1174
1175 cur = atomic_read(&f->rr_cur);
1176 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1177 fanout_rr_next(f, num))) != cur)
1178 cur = old;
77f65ebd
WB		 1179 return cur;
1180}
1181
1182static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1183 struct sk_buff *skb,
1184 unsigned int num)
1185{
1186 return smp_processor_id() % num;
dc99f600
DM		 1187}
1188
5df0ddfb
DB		 1189static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1190 struct sk_buff *skb,
1191 unsigned int num)
1192{
1193 return reciprocal_divide(prandom_u32(), num);
1194}
1195
77f65ebd
WB		 1196static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1197 struct sk_buff *skb,
1198 unsigned int idx, unsigned int skip,
1199 unsigned int num)
95ec3eb4 1200{
77f65ebd 1201 unsigned int i, j;
95ec3eb4 1202
77f65ebd
WB		 1203 i = j = min_t(int, f->next[idx], num - 1);
1204 do {
1205 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1206 if (i != j)
1207 f->next[idx] = i;
1208 return i;
1209 }
1210 if (++i == num)
1211 i = 0;
1212 } while (i != j);
1213
1214 return idx;
1215}
1216
1217static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1218{
1219 return f->flags & (flag >> 8);
95ec3eb4
DM		 1220}
1221
95ec3eb4
DM		 1222static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1223 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1224{
1225 struct packet_fanout *f = pt->af_packet_priv;
1226 unsigned int num = f->num_members;
1227 struct packet_sock *po;
77f65ebd 1228 unsigned int idx;
dc99f600
DM		 1229
1230 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1231 !num) {
1232 kfree_skb(skb);
1233 return 0;
1234 }
1235
95ec3eb4
DM		 1236 switch (f->type) {
1237 case PACKET_FANOUT_HASH:
1238 default:
77f65ebd 1239 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1240 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1241 if (!skb)
1242 return 0;
1243 }
1244 skb_get_rxhash(skb);
77f65ebd 1245 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1246 break;
1247 case PACKET_FANOUT_LB:
77f65ebd 1248 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1249 break;
1250 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1251 idx = fanout_demux_cpu(f, skb, num);
1252 break;
5df0ddfb
DB		 1253 case PACKET_FANOUT_RND:
1254 idx = fanout_demux_rnd(f, skb, num);
1255 break;
77f65ebd
WB		 1256 case PACKET_FANOUT_ROLLOVER:
1257 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1258 break;
dc99f600
DM		 1259 }
1260
77f65ebd
WB		 1261 po = pkt_sk(f->arr[idx]);
1262 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1263 unlikely(!packet_rcv_has_room(po, skb))) {
1264 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1265 po = pkt_sk(f->arr[idx]);
1266 }
dc99f600
DM		 1267
1268 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1269}
1270
fff3321d
PE		 1271DEFINE_MUTEX(fanout_mutex);
1272EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1273static LIST_HEAD(fanout_list);
1274
1275static void __fanout_link(struct sock *sk, struct packet_sock *po)
1276{
1277 struct packet_fanout *f = po->fanout;
1278
1279 spin_lock(&f->lock);
1280 f->arr[f->num_members] = sk;
1281 smp_wmb();
1282 f->num_members++;
1283 spin_unlock(&f->lock);
1284}
1285
1286static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1287{
1288 struct packet_fanout *f = po->fanout;
1289 int i;
1290
1291 spin_lock(&f->lock);
1292 for (i = 0; i < f->num_members; i++) {
1293 if (f->arr[i] == sk)
1294 break;
1295 }
1296 BUG_ON(i >= f->num_members);
1297 f->arr[i] = f->arr[f->num_members - 1];
1298 f->num_members--;
1299 spin_unlock(&f->lock);
1300}
1301
a0dfb263 1302static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1303{
1304 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1305 return true;
1306
1307 return false;
1308}
1309
7736d33f 1310static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1311{
1312 struct packet_sock *po = pkt_sk(sk);
1313 struct packet_fanout *f, *match;
7736d33f 1314 u8 type = type_flags & 0xff;
77f65ebd 1315 u8 flags = type_flags >> 8;
dc99f600
DM		 1316 int err;
1317
1318 switch (type) {
77f65ebd
WB		 1319 case PACKET_FANOUT_ROLLOVER:
1320 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1321 return -EINVAL;
dc99f600
DM		 1322 case PACKET_FANOUT_HASH:
1323 case PACKET_FANOUT_LB:
95ec3eb4 1324 case PACKET_FANOUT_CPU:
5df0ddfb 1325 case PACKET_FANOUT_RND:
dc99f600
DM		 1326 break;
1327 default:
1328 return -EINVAL;
1329 }
1330
1331 if (!po->running)
1332 return -EINVAL;
1333
1334 if (po->fanout)
1335 return -EALREADY;
1336
1337 mutex_lock(&fanout_mutex);
1338 match = NULL;
1339 list_for_each_entry(f, &fanout_list, list) {
1340 if (f->id == id &&
1341 read_pnet(&f->net) == sock_net(sk)) {
1342 match = f;
1343 break;
1344 }
1345 }
afe62c68 1346 err = -EINVAL;
77f65ebd 1347 if (match && match->flags != flags)
afe62c68 1348 goto out;
dc99f600 1349 if (!match) {
afe62c68 1350 err = -ENOMEM;
dc99f600 1351 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1352 if (!match)
1353 goto out;
1354 write_pnet(&match->net, sock_net(sk));
1355 match->id = id;
1356 match->type = type;
77f65ebd 1357 match->flags = flags;
afe62c68
ED		 1358 atomic_set(&match->rr_cur, 0);
1359 INIT_LIST_HEAD(&match->list);
1360 spin_lock_init(&match->lock);
1361 atomic_set(&match->sk_ref, 0);
1362 match->prot_hook.type = po->prot_hook.type;
1363 match->prot_hook.dev = po->prot_hook.dev;
1364 match->prot_hook.func = packet_rcv_fanout;
1365 match->prot_hook.af_packet_priv = match;
c0de08d0 1366 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1367 dev_add_pack(&match->prot_hook);
1368 list_add(&match->list, &fanout_list);
dc99f600 1369 }
afe62c68
ED		 1370 err = -EINVAL;
1371 if (match->type == type &&
1372 match->prot_hook.type == po->prot_hook.type &&
1373 match->prot_hook.dev == po->prot_hook.dev) {
1374 err = -ENOSPC;
1375 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1376 __dev_remove_pack(&po->prot_hook);
1377 po->fanout = match;
1378 atomic_inc(&match->sk_ref);
1379 __fanout_link(sk, po);
1380 err = 0;
dc99f600
DM		 1381 }
1382 }
afe62c68 1383out:
dc99f600
DM		 1384 mutex_unlock(&fanout_mutex);
1385 return err;
1386}
1387
1388static void fanout_release(struct sock *sk)
1389{
1390 struct packet_sock *po = pkt_sk(sk);
1391 struct packet_fanout *f;
1392
1393 f = po->fanout;
1394 if (!f)
1395 return;
1396
fff3321d 1397 mutex_lock(&fanout_mutex);
dc99f600
DM		 1398 po->fanout = NULL;
1399
dc99f600
DM		 1400 if (atomic_dec_and_test(&f->sk_ref)) {
1401 list_del(&f->list);
1402 dev_remove_pack(&f->prot_hook);
1403 kfree(f);
1404 }
1405 mutex_unlock(&fanout_mutex);
1406}
1da177e4 1407
90ddc4f0 1408static const struct proto_ops packet_ops;
1da177e4 1409
90ddc4f0 1410static const struct proto_ops packet_ops_spkt;
1da177e4 1411
40d4e3df
ED		 1412static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1413 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1414{
1415 struct sock *sk;
1416 struct sockaddr_pkt *spkt;
1417
1418 /*
1419 * When we registered the protocol we saved the socket in the data
1420 * field for just this event.
1421 */
1422
1423 sk = pt->af_packet_priv;
1ce4f28b 1424
1da177e4
LT		 1425 /*
1426 * Yank back the headers [hope the device set this
1427 * right or kerboom...]
1428 *
1429 * Incoming packets have ll header pulled,
1430 * push it back.
1431 *
98e399f8 1432 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1433 * so that this procedure is noop.
1434 */
1435
1436 if (skb->pkt_type == PACKET_LOOPBACK)
1437 goto out;
1438
09ad9bc7 1439 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1440 goto out;
1441
40d4e3df
ED		 1442 skb = skb_share_check(skb, GFP_ATOMIC);
1443 if (skb == NULL)
1da177e4
LT		 1444 goto oom;
1445
1446 /* drop any routing info */
adf30907 1447 skb_dst_drop(skb);
1da177e4 1448
84531c24
PO		 1449 /* drop conntrack reference */
1450 nf_reset(skb);
1451
ffbc6111 1452 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1453
98e399f8 1454 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1455
1456 /*
1457 * The SOCK_PACKET socket receives _all_ frames.
1458 */
1459
1460 spkt->spkt_family = dev->type;
1461 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1462 spkt->spkt_protocol = skb->protocol;
1463
1464 /*
1465 * Charge the memory to the socket. This is done specifically
1466 * to prevent sockets using all the memory up.
1467 */
1468
40d4e3df 1469 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1470 return 0;
1471
1472out:
1473 kfree_skb(skb);
1474oom:
1475 return 0;
1476}
1477
1478
1479/*
1480 * Output a raw packet to a device layer. This bypasses all the other
1481 * protocol layers and you must therefore supply it with a complete frame
1482 */
1ce4f28b 1483
1da177e4
LT		 1484static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1485 struct msghdr *msg, size_t len)
1486{
1487 struct sock *sk = sock->sk;
40d4e3df 1488 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1489 struct sk_buff *skb = NULL;
1da177e4 1490 struct net_device *dev;
40d4e3df 1491 __be16 proto = 0;
1da177e4 1492 int err;
3bdc0eba 1493 int extra_len = 0;
1ce4f28b 1494
1da177e4 1495 /*
1ce4f28b 1496 * Get and verify the address.
1da177e4
LT		 1497 */
1498
40d4e3df 1499 if (saddr) {
1da177e4 1500 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1501 return -EINVAL;
1502 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1503 proto = saddr->spkt_protocol;
1504 } else
1505 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1506
1507 /*
1ce4f28b 1508 * Find the device first to size check it
1da177e4
LT		 1509 */
1510
de74e92a 1511 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1512retry:
654d1f8a
ED		 1513 rcu_read_lock();
1514 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1515 err = -ENODEV;
1516 if (dev == NULL)
1517 goto out_unlock;
1ce4f28b 1518
d5e76b0a
DM		 1519 err = -ENETDOWN;
1520 if (!(dev->flags & IFF_UP))
1521 goto out_unlock;
1522
1da177e4 1523 /*
40d4e3df
ED		 1524 * You may not queue a frame bigger than the mtu. This is the lowest level
1525 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1526 */
1ce4f28b 1527
3bdc0eba
BG		 1528 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1529 if (!netif_supports_nofcs(dev)) {
1530 err = -EPROTONOSUPPORT;
1531 goto out_unlock;
1532 }
1533 extra_len = 4; /* We're doing our own CRC */
1534 }
1535
1da177e4 1536 err = -EMSGSIZE;
3bdc0eba 1537 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1538 goto out_unlock;
1539
1a35ca80
ED		 1540 if (!skb) {
1541 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1542 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1543 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1544
1545 rcu_read_unlock();
4ce40912 1546 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1547 if (skb == NULL)
1548 return -ENOBUFS;
1549 /* FIXME: Save some space for broken drivers that write a hard
1550 * header at transmission time by themselves. PPP is the notable
1551 * one here. This should really be fixed at the driver level.
1552 */
1553 skb_reserve(skb, reserved);
1554 skb_reset_network_header(skb);
1555
1556 /* Try to align data part correctly */
1557 if (hhlen) {
1558 skb->data -= hhlen;
1559 skb->tail -= hhlen;
1560 if (len < hhlen)
1561 skb_reset_network_header(skb);
1562 }
1563 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1564 if (err)
1565 goto out_free;
1566 goto retry;
1da177e4
LT		 1567 }
1568
3bdc0eba 1569 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1570 /* Earlier code assumed this would be a VLAN pkt,
1571 * double-check this now that we have the actual
1572 * packet in hand.
1573 */
1574 struct ethhdr *ehdr;
1575 skb_reset_mac_header(skb);
1576 ehdr = eth_hdr(skb);
1577 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1578 err = -EMSGSIZE;
1579 goto out_unlock;
1580 }
1581 }
1a35ca80 1582
1da177e4
LT		 1583 skb->protocol = proto;
1584 skb->dev = dev;
1585 skb->priority = sk->sk_priority;
2d37a186 1586 skb->mark = sk->sk_mark;
bf84a010
DB		 1587
1588 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1589
3bdc0eba
BG		 1590 if (unlikely(extra_len == 4))
1591 skb->no_fcs = 1;
1592
40893fd0 1593 skb_probe_transport_header(skb, 0);
c1aad275 1594
1da177e4 1595 dev_queue_xmit(skb);
654d1f8a 1596 rcu_read_unlock();
40d4e3df 1597 return len;
1da177e4 1598
1da177e4 1599out_unlock:
654d1f8a 1600 rcu_read_unlock();
1a35ca80
ED		 1601out_free:
1602 kfree_skb(skb);
1da177e4
LT		 1603 return err;
1604}
1da177e4 1605
eea49cc9 1606static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1607 const struct sock *sk,
dbcb5855 1608 unsigned int res)
1da177e4
LT		 1609{
1610 struct sk_filter *filter;
fda9ef5d 1611
80f8f102
ED		 1612 rcu_read_lock();
1613 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1614 if (filter != NULL)
0a14842f 1615 res = SK_RUN_FILTER(filter, skb);
80f8f102 1616 rcu_read_unlock();
1da177e4 1617
dbcb5855 1618 return res;
1da177e4
LT		 1619}
1620
1621/*
62ab0812
ED		 1622 * This function makes lazy skb cloning in hope that most of packets
1623 * are discarded by BPF.
1624 *
1625 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1626 * and skb->cb are mangled. It works because (and until) packets
1627 * falling here are owned by current CPU. Output packets are cloned
1628 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1629 * sequencially, so that if we return skb to original state on exit,
1630 * we will not harm anyone.
1da177e4
LT		 1631 */
1632
40d4e3df
ED		 1633static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1634 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1635{
1636 struct sock *sk;
1637 struct sockaddr_ll *sll;
1638 struct packet_sock *po;
40d4e3df 1639 u8 *skb_head = skb->data;
1da177e4 1640 int skb_len = skb->len;
dbcb5855 1641 unsigned int snaplen, res;
1da177e4
LT		 1642
1643 if (skb->pkt_type == PACKET_LOOPBACK)
1644 goto drop;
1645
1646 sk = pt->af_packet_priv;
1647 po = pkt_sk(sk);
1648
09ad9bc7 1649 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1650 goto drop;
1651
1da177e4
LT		 1652 skb->dev = dev;
1653
3b04ddde 1654 if (dev->header_ops) {
1da177e4 1655 /* The device has an explicit notion of ll header,
62ab0812
ED		 1656 * exported to higher levels.
1657 *
1658 * Otherwise, the device hides details of its frame
1659 * structure, so that corresponding packet head is
1660 * never delivered to user.
1da177e4
LT		 1661 */
1662 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1663 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1664 else if (skb->pkt_type == PACKET_OUTGOING) {
1665 /* Special case: outgoing packets have ll header at head */
bbe735e4 1666 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1667 }
1668 }
1669
1670 snaplen = skb->len;
1671
dbcb5855
DM		 1672 res = run_filter(skb, sk, snaplen);
1673 if (!res)
fda9ef5d 1674 goto drop_n_restore;
dbcb5855
DM		 1675 if (snaplen > res)
1676 snaplen = res;
1da177e4 1677
0fd7bac6 1678 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1679 goto drop_n_acct;
1680
1681 if (skb_shared(skb)) {
1682 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1683 if (nskb == NULL)
1684 goto drop_n_acct;
1685
1686 if (skb_head != skb->data) {
1687 skb->data = skb_head;
1688 skb->len = skb_len;
1689 }
abc4e4fa 1690 consume_skb(skb);
1da177e4
LT		 1691 skb = nskb;
1692 }
1693
ffbc6111
HX		 1694 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1695 sizeof(skb->cb));
1696
1697 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1698 sll->sll_family = AF_PACKET;
1699 sll->sll_hatype = dev->type;
1700 sll->sll_protocol = skb->protocol;
1701 sll->sll_pkttype = skb->pkt_type;
8032b464 1702 if (unlikely(po->origdev))
80feaacb
PWJ		 1703 sll->sll_ifindex = orig_dev->ifindex;
1704 else
1705 sll->sll_ifindex = dev->ifindex;
1da177e4 1706
b95cce35 1707 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1708
ffbc6111 1709 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1710
1da177e4
LT		 1711 if (pskb_trim(skb, snaplen))
1712 goto drop_n_acct;
1713
1714 skb_set_owner_r(skb, sk);
1715 skb->dev = NULL;
adf30907 1716 skb_dst_drop(skb);
1da177e4 1717
84531c24
PO		 1718 /* drop conntrack reference */
1719 nf_reset(skb);
1720
1da177e4 1721 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1722 po->stats.stats1.tp_packets++;
3b885787 1723 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1724 __skb_queue_tail(&sk->sk_receive_queue, skb);
1725 spin_unlock(&sk->sk_receive_queue.lock);
1726 sk->sk_data_ready(sk, skb->len);
1727 return 0;
1728
1729drop_n_acct:
7091fbd8 1730 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1731 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1732 atomic_inc(&sk->sk_drops);
1733 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1734
1735drop_n_restore:
1736 if (skb_head != skb->data && skb_shared(skb)) {
1737 skb->data = skb_head;
1738 skb->len = skb_len;
1739 }
1740drop:
ead2ceb0 1741 consume_skb(skb);
1da177e4
LT		 1742 return 0;
1743}
1744
40d4e3df
ED		 1745static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1746 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1747{
1748 struct sock *sk;
1749 struct packet_sock *po;
1750 struct sockaddr_ll *sll;
184f489e 1751 union tpacket_uhdr h;
40d4e3df 1752 u8 *skb_head = skb->data;
1da177e4 1753 int skb_len = skb->len;
dbcb5855 1754 unsigned int snaplen, res;
f6fb8f10 1755 unsigned long status = TP_STATUS_USER;
bbd6ef87 1756 unsigned short macoff, netoff, hdrlen;
1da177e4 1757 struct sk_buff *copy_skb = NULL;
bbd6ef87 1758 struct timespec ts;
b9c32fb2 1759 __u32 ts_status;
1da177e4
LT		 1760
1761 if (skb->pkt_type == PACKET_LOOPBACK)
1762 goto drop;
1763
1764 sk = pt->af_packet_priv;
1765 po = pkt_sk(sk);
1766
09ad9bc7 1767 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1768 goto drop;
1769
3b04ddde 1770 if (dev->header_ops) {
1da177e4 1771 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1772 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1773 else if (skb->pkt_type == PACKET_OUTGOING) {
1774 /* Special case: outgoing packets have ll header at head */
bbe735e4 1775 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1776 }
1777 }
1778
8dc41944
HX		 1779 if (skb->ip_summed == CHECKSUM_PARTIAL)
1780 status |= TP_STATUS_CSUMNOTREADY;
1781
1da177e4
LT		 1782 snaplen = skb->len;
1783
dbcb5855
DM		 1784 res = run_filter(skb, sk, snaplen);
1785 if (!res)
fda9ef5d 1786 goto drop_n_restore;
dbcb5855
DM		 1787 if (snaplen > res)
1788 snaplen = res;
1da177e4
LT		 1789
1790 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1791 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1792 po->tp_reserve;
1da177e4 1793 } else {
95c96174 1794 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1795 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1796 (maclen < 16 ? 16 : maclen)) +
1797 po->tp_reserve;
1da177e4
LT		 1798 macoff = netoff - maclen;
1799 }
f6fb8f10 1800 if (po->tp_version <= TPACKET_V2) {
1801 if (macoff + snaplen > po->rx_ring.frame_size) {
1802 if (po->copy_thresh &&
0fd7bac6 1803 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1804 if (skb_shared(skb)) {
1805 copy_skb = skb_clone(skb, GFP_ATOMIC);
1806 } else {
1807 copy_skb = skb_get(skb);
1808 skb_head = skb->data;
1809 }
1810 if (copy_skb)
1811 skb_set_owner_r(copy_skb, sk);
1da177e4 1812 }
f6fb8f10 1813 snaplen = po->rx_ring.frame_size - macoff;
1814 if ((int)snaplen < 0)
1815 snaplen = 0;
1da177e4 1816 }
1da177e4 1817 }
1da177e4 1818 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1819 h.raw = packet_current_rx_frame(po, skb,
1820 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1821 if (!h.raw)
1da177e4 1822 goto ring_is_full;
f6fb8f10 1823 if (po->tp_version <= TPACKET_V2) {
1824 packet_increment_rx_head(po, &po->rx_ring);
1825 /*
1826 * LOSING will be reported till you read the stats,
1827 * because it's COR - Clear On Read.
1828 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1829 * at packet level.
1830 */
ee80fbf3 1831 if (po->stats.stats1.tp_drops)
f6fb8f10 1832 status |= TP_STATUS_LOSING;
1833 }
ee80fbf3 1834 po->stats.stats1.tp_packets++;
1da177e4
LT		 1835 if (copy_skb) {
1836 status |= TP_STATUS_COPY;
1837 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1838 }
1da177e4
LT		 1839 spin_unlock(&sk->sk_receive_queue.lock);
1840
bbd6ef87 1841 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1842
1843 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1844 getnstimeofday(&ts);
1da177e4 1845
b9c32fb2
DB		 1846 status |= ts_status;
1847
bbd6ef87
PM		 1848 switch (po->tp_version) {
1849 case TPACKET_V1:
1850 h.h1->tp_len = skb->len;
1851 h.h1->tp_snaplen = snaplen;
1852 h.h1->tp_mac = macoff;
1853 h.h1->tp_net = netoff;
4b457bdf
DB		 1854 h.h1->tp_sec = ts.tv_sec;
1855 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1856 hdrlen = sizeof(*h.h1);
1857 break;
1858 case TPACKET_V2:
1859 h.h2->tp_len = skb->len;
1860 h.h2->tp_snaplen = snaplen;
1861 h.h2->tp_mac = macoff;
1862 h.h2->tp_net = netoff;
bbd6ef87
PM		 1863 h.h2->tp_sec = ts.tv_sec;
1864 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1865 if (vlan_tx_tag_present(skb)) {
1866 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1867 status |= TP_STATUS_VLAN_VALID;
1868 } else {
1869 h.h2->tp_vlan_tci = 0;
1870 }
13fcb7bd 1871 h.h2->tp_padding = 0;
bbd6ef87
PM		 1872 hdrlen = sizeof(*h.h2);
1873 break;
f6fb8f10 1874 case TPACKET_V3:
1875 /* tp_nxt_offset,vlan are already populated above.
1876 * So DONT clear those fields here
1877 */
1878 h.h3->tp_status |= status;
1879 h.h3->tp_len = skb->len;
1880 h.h3->tp_snaplen = snaplen;
1881 h.h3->tp_mac = macoff;
1882 h.h3->tp_net = netoff;
f6fb8f10 1883 h.h3->tp_sec = ts.tv_sec;
1884 h.h3->tp_nsec = ts.tv_nsec;
1885 hdrlen = sizeof(*h.h3);
1886 break;
bbd6ef87
PM		 1887 default:
1888 BUG();
1889 }
1da177e4 1890
bbd6ef87 1891 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1892 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1893 sll->sll_family = AF_PACKET;
1894 sll->sll_hatype = dev->type;
1895 sll->sll_protocol = skb->protocol;
1896 sll->sll_pkttype = skb->pkt_type;
8032b464 1897 if (unlikely(po->origdev))
80feaacb
PWJ		 1898 sll->sll_ifindex = orig_dev->ifindex;
1899 else
1900 sll->sll_ifindex = dev->ifindex;
1da177e4 1901
e16aa207 1902 smp_mb();
f6dafa95 1903#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1904 {
0af55bb5
CG		 1905 u8 *start, *end;
1906
f6fb8f10 1907 if (po->tp_version <= TPACKET_V2) {
1908 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1909 + macoff + snaplen);
1910 for (start = h.raw; start < end; start += PAGE_SIZE)
1911 flush_dcache_page(pgv_to_page(start));
1912 }
cc9f01b2 1913 smp_wmb();
1da177e4 1914 }
f6dafa95 1915#endif
f6fb8f10 1916 if (po->tp_version <= TPACKET_V2)
1917 __packet_set_status(po, h.raw, status);
1918 else
1919 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1920
1921 sk->sk_data_ready(sk, 0);
1922
1923drop_n_restore:
1924 if (skb_head != skb->data && skb_shared(skb)) {
1925 skb->data = skb_head;
1926 skb->len = skb_len;
1927 }
1928drop:
1ce4f28b 1929 kfree_skb(skb);
1da177e4
LT		 1930 return 0;
1931
1932ring_is_full:
ee80fbf3 1933 po->stats.stats1.tp_drops++;
1da177e4
LT		 1934 spin_unlock(&sk->sk_receive_queue.lock);
1935
1936 sk->sk_data_ready(sk, 0);
acb5d75b 1937 kfree_skb(copy_skb);
1da177e4
LT		 1938 goto drop_n_restore;
1939}
1940
69e3c75f
JB		 1941static void tpacket_destruct_skb(struct sk_buff *skb)
1942{
1943 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1944 void *ph;
1da177e4 1945
69e3c75f 1946 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 1947 __u32 ts;
1948
69e3c75f 1949 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1950 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1951 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 1952
1953 ts = __packet_set_timestamp(po, ph, skb);
1954 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 1955 }
1956
1957 sock_wfree(skb);
1958}
1959
40d4e3df
ED		 1960static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1961 void *frame, struct net_device *dev, int size_max,
ae641949 1962 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1963{
184f489e 1964 union tpacket_uhdr ph;
09effa67 1965 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 1966 struct socket *sock = po->sk.sk_socket;
1967 struct page *page;
1968 void *data;
1969 int err;
1970
1971 ph.raw = frame;
1972
1973 skb->protocol = proto;
1974 skb->dev = dev;
1975 skb->priority = po->sk.sk_priority;
2d37a186 1976 skb->mark = po->sk.sk_mark;
2e31396f 1977 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 1978 skb_shinfo(skb)->destructor_arg = ph.raw;
1979
1980 switch (po->tp_version) {
1981 case TPACKET_V2:
1982 tp_len = ph.h2->tp_len;
1983 break;
1984 default:
1985 tp_len = ph.h1->tp_len;
1986 break;
1987 }
09effa67
DM		 1988 if (unlikely(tp_len > size_max)) {
1989 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
1990 return -EMSGSIZE;
1991 }
69e3c75f 1992
ae641949 1993 skb_reserve(skb, hlen);
69e3c75f 1994 skb_reset_network_header(skb);
40893fd0 1995 skb_probe_transport_header(skb, 0);
c1aad275 1996
5920cd3a
PC		 1997 if (po->tp_tx_has_off) {
1998 int off_min, off_max, off;
1999 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2000 off_max = po->tx_ring.frame_size - tp_len;
2001 if (sock->type == SOCK_DGRAM) {
2002 switch (po->tp_version) {
2003 case TPACKET_V2:
2004 off = ph.h2->tp_net;
2005 break;
2006 default:
2007 off = ph.h1->tp_net;
2008 break;
2009 }
2010 } else {
2011 switch (po->tp_version) {
2012 case TPACKET_V2:
2013 off = ph.h2->tp_mac;
2014 break;
2015 default:
2016 off = ph.h1->tp_mac;
2017 break;
2018 }
2019 }
2020 if (unlikely((off < off_min) || (off_max < off)))
2021 return -EINVAL;
2022 data = ph.raw + off;
2023 } else {
2024 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2025 }
69e3c75f
JB		 2026 to_write = tp_len;
2027
2028 if (sock->type == SOCK_DGRAM) {
2029 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2030 NULL, tp_len);
2031 if (unlikely(err < 0))
2032 return -EINVAL;
40d4e3df 2033 } else if (dev->hard_header_len) {
69e3c75f
JB		 2034 /* net device doesn't like empty head */
2035 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2036 pr_err("packet size is too short (%d < %d)\n",
2037 tp_len, dev->hard_header_len);
69e3c75f
JB		 2038 return -EINVAL;
2039 }
2040
2041 skb_push(skb, dev->hard_header_len);
2042 err = skb_store_bits(skb, 0, data,
2043 dev->hard_header_len);
2044 if (unlikely(err))
2045 return err;
2046
2047 data += dev->hard_header_len;
2048 to_write -= dev->hard_header_len;
2049 }
2050
69e3c75f
JB		 2051 offset = offset_in_page(data);
2052 len_max = PAGE_SIZE - offset;
2053 len = ((to_write > len_max) ? len_max : to_write);
2054
2055 skb->data_len = to_write;
2056 skb->len += to_write;
2057 skb->truesize += to_write;
2058 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2059
2060 while (likely(to_write)) {
2061 nr_frags = skb_shinfo(skb)->nr_frags;
2062
2063 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2064 pr_err("Packet exceed the number of skb frags(%lu)\n",
2065 MAX_SKB_FRAGS);
69e3c75f
JB		 2066 return -EFAULT;
2067 }
2068
0af55bb5
CG		 2069 page = pgv_to_page(data);
2070 data += len;
69e3c75f
JB		 2071 flush_dcache_page(page);
2072 get_page(page);
0af55bb5 2073 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2074 to_write -= len;
2075 offset = 0;
2076 len_max = PAGE_SIZE;
2077 len = ((to_write > len_max) ? len_max : to_write);
2078 }
2079
2080 return tp_len;
2081}
2082
2083static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2084{
69e3c75f
JB		 2085 struct sk_buff *skb;
2086 struct net_device *dev;
2087 __be16 proto;
09effa67 2088 int err, reserve = 0;
40d4e3df
ED		 2089 void *ph;
2090 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2091 int tp_len, size_max;
2092 unsigned char *addr;
2093 int len_sum = 0;
9e67030a 2094 int status = TP_STATUS_AVAILABLE;
ae641949 2095 int hlen, tlen;
69e3c75f 2096
69e3c75f
JB		 2097 mutex_lock(&po->pg_vec_lock);
2098
66e56cd4 2099 if (likely(saddr == NULL)) {
e40526cb 2100 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2101 proto = po->num;
2102 addr = NULL;
2103 } else {
2104 err = -EINVAL;
2105 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2106 goto out;
2107 if (msg->msg_namelen < (saddr->sll_halen
2108 + offsetof(struct sockaddr_ll,
2109 sll_addr)))
2110 goto out;
69e3c75f
JB		 2111 proto = saddr->sll_protocol;
2112 addr = saddr->sll_addr;
827d9780 2113 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2114 }
2115
69e3c75f
JB		 2116 err = -ENXIO;
2117 if (unlikely(dev == NULL))
2118 goto out;
69e3c75f
JB		 2119 err = -ENETDOWN;
2120 if (unlikely(!(dev->flags & IFF_UP)))
2121 goto out_put;
2122
e40526cb
DB		 2123 reserve = dev->hard_header_len;
2124
69e3c75f 2125 size_max = po->tx_ring.frame_size
b5dd884e 2126 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2127
09effa67
DM		 2128 if (size_max > dev->mtu + reserve)
2129 size_max = dev->mtu + reserve;
2130
69e3c75f
JB		 2131 do {
2132 ph = packet_current_frame(po, &po->tx_ring,
2133 TP_STATUS_SEND_REQUEST);
2134
2135 if (unlikely(ph == NULL)) {
2136 schedule();
2137 continue;
2138 }
2139
2140 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2141 hlen = LL_RESERVED_SPACE(dev);
2142 tlen = dev->needed_tailroom;
69e3c75f 2143 skb = sock_alloc_send_skb(&po->sk,
ae641949 2144 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2145 0, &err);
2146
2147 if (unlikely(skb == NULL))
2148 goto out_status;
2149
2150 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2151 addr, hlen);
69e3c75f
JB		 2152
2153 if (unlikely(tp_len < 0)) {
2154 if (po->tp_loss) {
2155 __packet_set_status(po, ph,
2156 TP_STATUS_AVAILABLE);
2157 packet_increment_head(&po->tx_ring);
2158 kfree_skb(skb);
2159 continue;
2160 } else {
2161 status = TP_STATUS_WRONG_FORMAT;
2162 err = tp_len;
2163 goto out_status;
2164 }
2165 }
2166
2167 skb->destructor = tpacket_destruct_skb;
2168 __packet_set_status(po, ph, TP_STATUS_SENDING);
2169 atomic_inc(&po->tx_ring.pending);
2170
2171 status = TP_STATUS_SEND_REQUEST;
2172 err = dev_queue_xmit(skb);
eb70df13
JP		 2173 if (unlikely(err > 0)) {
2174 err = net_xmit_errno(err);
2175 if (err && __packet_get_status(po, ph) ==
2176 TP_STATUS_AVAILABLE) {
2177 /* skb was destructed already */
2178 skb = NULL;
2179 goto out_status;
2180 }
2181 /*
2182 * skb was dropped but not destructed yet;
2183 * let's treat it like congestion or err < 0
2184 */
2185 err = 0;
2186 }
69e3c75f
JB		 2187 packet_increment_head(&po->tx_ring);
2188 len_sum += tp_len;
f64f9e71
JP		 2189 } while (likely((ph != NULL) ||
2190 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2191 (atomic_read(&po->tx_ring.pending))))
2192 );
69e3c75f
JB		 2193
2194 err = len_sum;
2195 goto out_put;
2196
69e3c75f
JB		 2197out_status:
2198 __packet_set_status(po, ph, status);
2199 kfree_skb(skb);
2200out_put:
e40526cb 2201 dev_put(dev);
69e3c75f
JB		 2202out:
2203 mutex_unlock(&po->pg_vec_lock);
2204 return err;
2205}
69e3c75f 2206
eea49cc9
OJ		 2207static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2208 size_t reserve, size_t len,
2209 size_t linear, int noblock,
2210 int *err)
bfd5f4a3
SS		 2211{
2212 struct sk_buff *skb;
2213
2214 /* Under a page? Don't bother with paged skb. */
2215 if (prepad + len < PAGE_SIZE || !linear)
2216 linear = len;
2217
2218 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2219 err, 0);
bfd5f4a3
SS		 2220 if (!skb)
2221 return NULL;
2222
2223 skb_reserve(skb, reserve);
2224 skb_put(skb, linear);
2225 skb->data_len = len - linear;
2226 skb->len += len - linear;
2227
2228 return skb;
2229}
2230
69e3c75f 2231static int packet_snd(struct socket *sock,
1da177e4
LT		 2232 struct msghdr *msg, size_t len)
2233{
2234 struct sock *sk = sock->sk;
40d4e3df 2235 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2236 struct sk_buff *skb;
2237 struct net_device *dev;
0e11c91e 2238 __be16 proto;
1da177e4 2239 unsigned char *addr;
827d9780 2240 int err, reserve = 0;
bfd5f4a3
SS		 2241 struct virtio_net_hdr vnet_hdr = { 0 };
2242 int offset = 0;
2243 int vnet_hdr_len;
2244 struct packet_sock *po = pkt_sk(sk);
2245 unsigned short gso_type = 0;
ae641949 2246 int hlen, tlen;
3bdc0eba 2247 int extra_len = 0;
1da177e4
LT		 2248
2249 /*
1ce4f28b 2250 * Get and verify the address.
1da177e4 2251 */
1ce4f28b 2252
66e56cd4 2253 if (likely(saddr == NULL)) {
e40526cb 2254 dev = packet_cached_dev_get(po);
1da177e4
LT		 2255 proto = po->num;
2256 addr = NULL;
2257 } else {
2258 err = -EINVAL;
2259 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2260 goto out;
0fb375fb
EB		 2261 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2262 goto out;
1da177e4
LT		 2263 proto = saddr->sll_protocol;
2264 addr = saddr->sll_addr;
827d9780 2265 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2266 }
2267
1da177e4 2268 err = -ENXIO;
e40526cb 2269 if (unlikely(dev == NULL))
1da177e4 2270 goto out_unlock;
d5e76b0a 2271 err = -ENETDOWN;
e40526cb 2272 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2273 goto out_unlock;
2274
e40526cb
DB		 2275 if (sock->type == SOCK_RAW)
2276 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2277 if (po->has_vnet_hdr) {
2278 vnet_hdr_len = sizeof(vnet_hdr);
2279
2280 err = -EINVAL;
2281 if (len < vnet_hdr_len)
2282 goto out_unlock;
2283
2284 len -= vnet_hdr_len;
2285
2286 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2287 vnet_hdr_len);
2288 if (err < 0)
2289 goto out_unlock;
2290
2291 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2292 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2293 vnet_hdr.hdr_len))
2294 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2295 vnet_hdr.csum_offset + 2;
2296
2297 err = -EINVAL;
2298 if (vnet_hdr.hdr_len > len)
2299 goto out_unlock;
2300
2301 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2302 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2303 case VIRTIO_NET_HDR_GSO_TCPV4:
2304 gso_type = SKB_GSO_TCPV4;
2305 break;
2306 case VIRTIO_NET_HDR_GSO_TCPV6:
2307 gso_type = SKB_GSO_TCPV6;
2308 break;
2309 case VIRTIO_NET_HDR_GSO_UDP:
2310 gso_type = SKB_GSO_UDP;
2311 break;
2312 default:
2313 goto out_unlock;
2314 }
2315
2316 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2317 gso_type |= SKB_GSO_TCP_ECN;
2318
2319 if (vnet_hdr.gso_size == 0)
2320 goto out_unlock;
2321
2322 }
2323 }
2324
3bdc0eba
BG		 2325 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2326 if (!netif_supports_nofcs(dev)) {
2327 err = -EPROTONOSUPPORT;
2328 goto out_unlock;
2329 }
2330 extra_len = 4; /* We're doing our own CRC */
2331 }
2332
1da177e4 2333 err = -EMSGSIZE;
3bdc0eba 2334 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2335 goto out_unlock;
2336
bfd5f4a3 2337 err = -ENOBUFS;
ae641949
HX		 2338 hlen = LL_RESERVED_SPACE(dev);
2339 tlen = dev->needed_tailroom;
2340 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2341 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2342 if (skb == NULL)
1da177e4
LT		 2343 goto out_unlock;
2344
bfd5f4a3 2345 skb_set_network_header(skb, reserve);
1da177e4 2346
0c4e8581
SH		 2347 err = -EINVAL;
2348 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2349 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2350 goto out_free;
1da177e4
LT		 2351
2352 /* Returns -EFAULT on error */
bfd5f4a3 2353 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2354 if (err)
2355 goto out_free;
bf84a010
DB		 2356
2357 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2358
3bdc0eba 2359 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2360 /* Earlier code assumed this would be a VLAN pkt,
2361 * double-check this now that we have the actual
2362 * packet in hand.
2363 */
2364 struct ethhdr *ehdr;
2365 skb_reset_mac_header(skb);
2366 ehdr = eth_hdr(skb);
2367 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2368 err = -EMSGSIZE;
2369 goto out_free;
2370 }
57f89bfa
BG		 2371 }
2372
09effa67
DM		 2373 skb->protocol = proto;
2374 skb->dev = dev;
1da177e4 2375 skb->priority = sk->sk_priority;
2d37a186 2376 skb->mark = sk->sk_mark;
1da177e4 2377
bfd5f4a3
SS		 2378 if (po->has_vnet_hdr) {
2379 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2380 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2381 vnet_hdr.csum_offset)) {
2382 err = -EINVAL;
2383 goto out_free;
2384 }
2385 }
2386
2387 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2388 skb_shinfo(skb)->gso_type = gso_type;
2389
2390 /* Header must be checked, and gso_segs computed. */
2391 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2392 skb_shinfo(skb)->gso_segs = 0;
2393
2394 len += vnet_hdr_len;
2395 }
2396
40893fd0 2397 skb_probe_transport_header(skb, reserve);
c1aad275 2398
3bdc0eba
BG		 2399 if (unlikely(extra_len == 4))
2400 skb->no_fcs = 1;
2401
1da177e4
LT		 2402 /*
2403 * Now send it
2404 */
2405
2406 err = dev_queue_xmit(skb);
2407 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2408 goto out_unlock;
2409
e40526cb 2410 dev_put(dev);
1da177e4 2411
40d4e3df 2412 return len;
1da177e4
LT		 2413
2414out_free:
2415 kfree_skb(skb);
2416out_unlock:
e40526cb 2417 if (dev)
1da177e4
LT		 2418 dev_put(dev);
2419out:
2420 return err;
2421}
2422
69e3c75f
JB		 2423static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2424 struct msghdr *msg, size_t len)
2425{
69e3c75f
JB		 2426 struct sock *sk = sock->sk;
2427 struct packet_sock *po = pkt_sk(sk);
2428 if (po->tx_ring.pg_vec)
2429 return tpacket_snd(po, msg);
2430 else
69e3c75f
JB		 2431 return packet_snd(sock, msg, len);
2432}
2433
1da177e4
LT		 2434/*
2435 * Close a PACKET socket. This is fairly simple. We immediately go
2436 * to 'closed' state and remove our protocol entry in the device list.
2437 */
2438
2439static int packet_release(struct socket *sock)
2440{
2441 struct sock *sk = sock->sk;
2442 struct packet_sock *po;
d12d01d6 2443 struct net *net;
f6fb8f10 2444 union tpacket_req_u req_u;
1da177e4
LT		 2445
2446 if (!sk)
2447 return 0;
2448
3b1e0a65 2449 net = sock_net(sk);
1da177e4
LT		 2450 po = pkt_sk(sk);
2451
0fa7fa98 2452 mutex_lock(&net->packet.sklist_lock);
808f5114 2453 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2454 mutex_unlock(&net->packet.sklist_lock);
2455
2456 preempt_disable();
920de804 2457 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2458 preempt_enable();
1da177e4 2459
808f5114 2460 spin_lock(&po->bind_lock);
ce06b03e 2461 unregister_prot_hook(sk, false);
66e56cd4
DB		 2462 packet_cached_dev_reset(po);
2463
160ff18a
BG		 2464 if (po->prot_hook.dev) {
2465 dev_put(po->prot_hook.dev);
2466 po->prot_hook.dev = NULL;
2467 }
808f5114 2468 spin_unlock(&po->bind_lock);
1da177e4 2469
1da177e4 2470 packet_flush_mclist(sk);
1da177e4 2471
9665d5d6
PS		 2472 if (po->rx_ring.pg_vec) {
2473 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2474 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2475 }
69e3c75f 2476
9665d5d6
PS		 2477 if (po->tx_ring.pg_vec) {
2478 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2479 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2480 }
1da177e4 2481
dc99f600
DM		 2482 fanout_release(sk);
2483
808f5114 2484 synchronize_net();
1da177e4
LT		 2485 /*
2486 * Now the socket is dead. No more input will appear.
2487 */
1da177e4
LT		 2488 sock_orphan(sk);
2489 sock->sk = NULL;
2490
2491 /* Purge queues */
2492
2493 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2494 sk_refcnt_debug_release(sk);
1da177e4
LT		 2495
2496 sock_put(sk);
2497 return 0;
2498}
2499
2500/*
2501 * Attach a packet hook.
2502 */
2503
0e11c91e 2504static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2505{
2506 struct packet_sock *po = pkt_sk(sk);
dc99f600 2507
aef950b4
WY		 2508 if (po->fanout) {
2509 if (dev)
2510 dev_put(dev);
2511
dc99f600 2512 return -EINVAL;
aef950b4 2513 }
1da177e4
LT		 2514
2515 lock_sock(sk);
2516
2517 spin_lock(&po->bind_lock);
ce06b03e 2518 unregister_prot_hook(sk, true);
66e56cd4 2519
1da177e4
LT		 2520 po->num = protocol;
2521 po->prot_hook.type = protocol;
160ff18a
BG		 2522 if (po->prot_hook.dev)
2523 dev_put(po->prot_hook.dev);
1da177e4 2524
66e56cd4 2525 po->prot_hook.dev = dev;
1da177e4
LT		 2526 po->ifindex = dev ? dev->ifindex : 0;
2527
66e56cd4
DB		 2528 packet_cached_dev_assign(po, dev);
2529
1da177e4
LT		 2530 if (protocol == 0)
2531 goto out_unlock;
2532
be85d4ad 2533 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2534 register_prot_hook(sk);
be85d4ad
UT		 2535 } else {
2536 sk->sk_err = ENETDOWN;
2537 if (!sock_flag(sk, SOCK_DEAD))
2538 sk->sk_error_report(sk);
1da177e4
LT		 2539 }
2540
2541out_unlock:
2542 spin_unlock(&po->bind_lock);
2543 release_sock(sk);
2544 return 0;
2545}
2546
2547/*
2548 * Bind a packet socket to a device
2549 */
2550
40d4e3df
ED		 2551static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2552 int addr_len)
1da177e4 2553{
40d4e3df 2554 struct sock *sk = sock->sk;
1da177e4
LT		 2555 char name[15];
2556 struct net_device *dev;
2557 int err = -ENODEV;
1ce4f28b 2558
1da177e4
LT		 2559 /*
2560 * Check legality
2561 */
1ce4f28b 2562
8ae55f04 2563 if (addr_len != sizeof(struct sockaddr))
1da177e4 2564 return -EINVAL;
40d4e3df 2565 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2566
3b1e0a65 2567 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2568 if (dev)
1da177e4 2569 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2570 return err;
2571}
1da177e4
LT		 2572
2573static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2574{
40d4e3df
ED		 2575 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2576 struct sock *sk = sock->sk;
1da177e4
LT		 2577 struct net_device *dev = NULL;
2578 int err;
2579
2580
2581 /*
2582 * Check legality
2583 */
1ce4f28b 2584
1da177e4
LT		 2585 if (addr_len < sizeof(struct sockaddr_ll))
2586 return -EINVAL;
2587 if (sll->sll_family != AF_PACKET)
2588 return -EINVAL;
2589
2590 if (sll->sll_ifindex) {
2591 err = -ENODEV;
3b1e0a65 2592 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2593 if (dev == NULL)
2594 goto out;
2595 }
2596 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2597
2598out:
2599 return err;
2600}
2601
2602static struct proto packet_proto = {
2603 .name = "PACKET",
2604 .owner = THIS_MODULE,
2605 .obj_size = sizeof(struct packet_sock),
2606};
2607
2608/*
1ce4f28b 2609 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2610 */
2611
3f378b68
EP		 2612static int packet_create(struct net *net, struct socket *sock, int protocol,
2613 int kern)
1da177e4
LT		 2614{
2615 struct sock *sk;
2616 struct packet_sock *po;
0e11c91e 2617 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2618 int err;
2619
df008c91 2620 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2621 return -EPERM;
be02097c
DM		 2622 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2623 sock->type != SOCK_PACKET)
1da177e4
LT		 2624 return -ESOCKTNOSUPPORT;
2625
2626 sock->state = SS_UNCONNECTED;
2627
2628 err = -ENOBUFS;
6257ff21 2629 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2630 if (sk == NULL)
2631 goto out;
2632
2633 sock->ops = &packet_ops;
1da177e4
LT		 2634 if (sock->type == SOCK_PACKET)
2635 sock->ops = &packet_ops_spkt;
be02097c 2636
1da177e4
LT		 2637 sock_init_data(sock, sk);
2638
2639 po = pkt_sk(sk);
2640 sk->sk_family = PF_PACKET;
0e11c91e 2641 po->num = proto;
66e56cd4
DB		 2642
2643 packet_cached_dev_reset(po);
1da177e4
LT		 2644
2645 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2646 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2647
2648 /*
2649 * Attach a protocol block
2650 */
2651
2652 spin_lock_init(&po->bind_lock);
905db440 2653 mutex_init(&po->pg_vec_lock);
1da177e4 2654 po->prot_hook.func = packet_rcv;
be02097c 2655
1da177e4
LT		 2656 if (sock->type == SOCK_PACKET)
2657 po->prot_hook.func = packet_rcv_spkt;
be02097c 2658
1da177e4
LT		 2659 po->prot_hook.af_packet_priv = sk;
2660
0e11c91e
AV		 2661 if (proto) {
2662 po->prot_hook.type = proto;
ce06b03e 2663 register_prot_hook(sk);
1da177e4
LT		 2664 }
2665
0fa7fa98 2666 mutex_lock(&net->packet.sklist_lock);
808f5114 2667 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2668 mutex_unlock(&net->packet.sklist_lock);
2669
2670 preempt_disable();
3680453c 2671 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2672 preempt_enable();
808f5114 2673
40d4e3df 2674 return 0;
1da177e4
LT		 2675out:
2676 return err;
2677}
2678
2679/*
2680 * Pull a packet from our receive queue and hand it to the user.
2681 * If necessary we block.
2682 */
2683
2684static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2685 struct msghdr *msg, size_t len, int flags)
2686{
2687 struct sock *sk = sock->sk;
2688 struct sk_buff *skb;
2689 int copied, err;
bfd5f4a3 2690 int vnet_hdr_len = 0;
1da177e4
LT		 2691
2692 err = -EINVAL;
ed85b565 2693 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2694 goto out;
2695
2696#if 0
2697 /* What error should we return now? EUNATTACH? */
2698 if (pkt_sk(sk)->ifindex < 0)
2699 return -ENODEV;
2700#endif
2701
ed85b565 2702 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2703 err = sock_recv_errqueue(sk, msg, len,
2704 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2705 goto out;
2706 }
2707
1da177e4
LT		 2708 /*
2709 * Call the generic datagram receiver. This handles all sorts
2710 * of horrible races and re-entrancy so we can forget about it
2711 * in the protocol layers.
2712 *
2713 * Now it will return ENETDOWN, if device have just gone down,
2714 * but then it will block.
2715 */
2716
40d4e3df 2717 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2718
2719 /*
1ce4f28b 2720 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2721 * handles the blocking we don't see and worry about blocking
2722 * retries.
2723 */
2724
8ae55f04 2725 if (skb == NULL)
1da177e4
LT		 2726 goto out;
2727
bfd5f4a3
SS		 2728 if (pkt_sk(sk)->has_vnet_hdr) {
2729 struct virtio_net_hdr vnet_hdr = { 0 };
2730
2731 err = -EINVAL;
2732 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2733 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2734 goto out_free;
2735
1f18b717
MK		 2736 len -= vnet_hdr_len;
2737
bfd5f4a3
SS		 2738 if (skb_is_gso(skb)) {
2739 struct skb_shared_info *sinfo = skb_shinfo(skb);
2740
2741 /* This is a hint as to how much should be linear. */
2742 vnet_hdr.hdr_len = skb_headlen(skb);
2743 vnet_hdr.gso_size = sinfo->gso_size;
2744 if (sinfo->gso_type & SKB_GSO_TCPV4)
2745 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2746 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2747 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2748 else if (sinfo->gso_type & SKB_GSO_UDP)
2749 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2750 else if (sinfo->gso_type & SKB_GSO_FCOE)
2751 goto out_free;
2752 else
2753 BUG();
2754 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2755 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2756 } else
2757 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2758
2759 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2760 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2761 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2762 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2763 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2764 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2765 } /* else everything is zero */
2766
2767 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2768 vnet_hdr_len);
2769 if (err < 0)
2770 goto out_free;
2771 }
2772
f3d33426
HFS		 2773 /* You lose any data beyond the buffer you gave. If it worries
2774 * a user program they can ask the device for its MTU
2775 * anyway.
1da177e4 2776 */
1da177e4 2777 copied = skb->len;
40d4e3df
ED		 2778 if (copied > len) {
2779 copied = len;
2780 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2781 }
2782
2783 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2784 if (err)
2785 goto out_free;
2786
3b885787 2787 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2788
f3d33426
HFS		 2789 if (msg->msg_name) {
2790 /* If the address length field is there to be filled
2791 * in, we fill it in now.
2792 */
2793 if (sock->type == SOCK_PACKET) {
2794 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2795 } else {
2796 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2797 msg->msg_namelen = sll->sll_halen +
2798 offsetof(struct sockaddr_ll, sll_addr);
2799 }
ffbc6111
HX		 2800 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2801 msg->msg_namelen);
f3d33426 2802 }
1da177e4 2803
8dc41944 2804 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2805 struct tpacket_auxdata aux;
2806
2807 aux.tp_status = TP_STATUS_USER;
2808 if (skb->ip_summed == CHECKSUM_PARTIAL)
2809 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2810 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2811 aux.tp_snaplen = skb->len;
2812 aux.tp_mac = 0;
bbe735e4 2813 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2814 if (vlan_tx_tag_present(skb)) {
2815 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2816 aux.tp_status |= TP_STATUS_VLAN_VALID;
2817 } else {
2818 aux.tp_vlan_tci = 0;
2819 }
13fcb7bd 2820 aux.tp_padding = 0;
ffbc6111 2821 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2822 }
2823
1da177e4
LT		 2824 /*
2825 * Free or return the buffer as appropriate. Again this
2826 * hides all the races and re-entrancy issues from us.
2827 */
bfd5f4a3 2828 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2829
2830out_free:
2831 skb_free_datagram(sk, skb);
2832out:
2833 return err;
2834}
2835
1da177e4
LT		 2836static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2837 int *uaddr_len, int peer)
2838{
2839 struct net_device *dev;
2840 struct sock *sk = sock->sk;
2841
2842 if (peer)
2843 return -EOPNOTSUPP;
2844
2845 uaddr->sa_family = AF_PACKET;
2dc85bf3 2846 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2847 rcu_read_lock();
2848 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2849 if (dev)
2dc85bf3 2850 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2851 rcu_read_unlock();
1da177e4
LT		 2852 *uaddr_len = sizeof(*uaddr);
2853
2854 return 0;
2855}
1da177e4
LT		 2856
2857static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2858 int *uaddr_len, int peer)
2859{
2860 struct net_device *dev;
2861 struct sock *sk = sock->sk;
2862 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2863 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2864
2865 if (peer)
2866 return -EOPNOTSUPP;
2867
2868 sll->sll_family = AF_PACKET;
2869 sll->sll_ifindex = po->ifindex;
2870 sll->sll_protocol = po->num;
67286640 2871 sll->sll_pkttype = 0;
654d1f8a
ED		 2872 rcu_read_lock();
2873 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2874 if (dev) {
2875 sll->sll_hatype = dev->type;
2876 sll->sll_halen = dev->addr_len;
2877 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2878 } else {
2879 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2880 sll->sll_halen = 0;
2881 }
654d1f8a 2882 rcu_read_unlock();
0fb375fb 2883 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2884
2885 return 0;
2886}
2887
2aeb0b88
WC		 2888static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2889 int what)
1da177e4
LT		 2890{
2891 switch (i->type) {
2892 case PACKET_MR_MULTICAST:
1162563f
JP		 2893 if (i->alen != dev->addr_len)
2894 return -EINVAL;
1da177e4 2895 if (what > 0)
22bedad3 2896 return dev_mc_add(dev, i->addr);
1da177e4 2897 else
22bedad3 2898 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2899 break;
2900 case PACKET_MR_PROMISC:
2aeb0b88 2901 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2902 break;
2903 case PACKET_MR_ALLMULTI:
2aeb0b88 2904 return dev_set_allmulti(dev, what);
1da177e4 2905 break;
d95ed927 2906 case PACKET_MR_UNICAST:
1162563f
JP		 2907 if (i->alen != dev->addr_len)
2908 return -EINVAL;
d95ed927 2909 if (what > 0)
a748ee24 2910 return dev_uc_add(dev, i->addr);
d95ed927 2911 else
a748ee24 2912 return dev_uc_del(dev, i->addr);
d95ed927 2913 break;
40d4e3df
ED		 2914 default:
2915 break;
1da177e4 2916 }
2aeb0b88 2917 return 0;
1da177e4
LT		 2918}
2919
2920static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2921{
40d4e3df 2922 for ( ; i; i = i->next) {
1da177e4
LT		 2923 if (i->ifindex == dev->ifindex)
2924 packet_dev_mc(dev, i, what);
2925 }
2926}
2927
0fb375fb 2928static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2929{
2930 struct packet_sock *po = pkt_sk(sk);
2931 struct packet_mclist *ml, *i;
2932 struct net_device *dev;
2933 int err;
2934
2935 rtnl_lock();
2936
2937 err = -ENODEV;
3b1e0a65 2938 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2939 if (!dev)
2940 goto done;
2941
2942 err = -EINVAL;
1162563f 2943 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2944 goto done;
2945
2946 err = -ENOBUFS;
8b3a7005 2947 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2948 if (i == NULL)
2949 goto done;
2950
2951 err = 0;
2952 for (ml = po->mclist; ml; ml = ml->next) {
2953 if (ml->ifindex == mreq->mr_ifindex &&
2954 ml->type == mreq->mr_type &&
2955 ml->alen == mreq->mr_alen &&
2956 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2957 ml->count++;
2958 /* Free the new element ... */
2959 kfree(i);
2960 goto done;
2961 }
2962 }
2963
2964 i->type = mreq->mr_type;
2965 i->ifindex = mreq->mr_ifindex;
2966 i->alen = mreq->mr_alen;
2967 memcpy(i->addr, mreq->mr_address, i->alen);
2968 i->count = 1;
2969 i->next = po->mclist;
2970 po->mclist = i;
2aeb0b88
WC		 2971 err = packet_dev_mc(dev, i, 1);
2972 if (err) {
2973 po->mclist = i->next;
2974 kfree(i);
2975 }
1da177e4
LT		 2976
2977done:
2978 rtnl_unlock();
2979 return err;
2980}
2981
0fb375fb 2982static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2983{
2984 struct packet_mclist *ml, **mlp;
2985
2986 rtnl_lock();
2987
2988 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2989 if (ml->ifindex == mreq->mr_ifindex &&
2990 ml->type == mreq->mr_type &&
2991 ml->alen == mreq->mr_alen &&
2992 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2993 if (--ml->count == 0) {
2994 struct net_device *dev;
2995 *mlp = ml->next;
ad959e76
ED		 2996 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2997 if (dev)
1da177e4 2998 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2999 kfree(ml);
3000 }
3001 rtnl_unlock();
3002 return 0;
3003 }
3004 }
3005 rtnl_unlock();
3006 return -EADDRNOTAVAIL;
3007}
3008
3009static void packet_flush_mclist(struct sock *sk)
3010{
3011 struct packet_sock *po = pkt_sk(sk);
3012 struct packet_mclist *ml;
3013
3014 if (!po->mclist)
3015 return;
3016
3017 rtnl_lock();
3018 while ((ml = po->mclist) != NULL) {
3019 struct net_device *dev;
3020
3021 po->mclist = ml->next;
ad959e76
ED		 3022 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3023 if (dev != NULL)
1da177e4 3024 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3025 kfree(ml);
3026 }
3027 rtnl_unlock();
3028}
1da177e4
LT		 3029
3030static int
b7058842 3031packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3032{
3033 struct sock *sk = sock->sk;
8dc41944 3034 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3035 int ret;
3036
3037 if (level != SOL_PACKET)
3038 return -ENOPROTOOPT;
3039
69e3c75f 3040 switch (optname) {
1ce4f28b 3041 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3042 case PACKET_DROP_MEMBERSHIP:
3043 {
0fb375fb
EB		 3044 struct packet_mreq_max mreq;
3045 int len = optlen;
3046 memset(&mreq, 0, sizeof(mreq));
3047 if (len < sizeof(struct packet_mreq))
1da177e4 3048 return -EINVAL;
0fb375fb
EB		 3049 if (len > sizeof(mreq))
3050 len = sizeof(mreq);
40d4e3df 3051 if (copy_from_user(&mreq, optval, len))
1da177e4 3052 return -EFAULT;
0fb375fb
EB		 3053 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3054 return -EINVAL;
1da177e4
LT		 3055 if (optname == PACKET_ADD_MEMBERSHIP)
3056 ret = packet_mc_add(sk, &mreq);
3057 else
3058 ret = packet_mc_drop(sk, &mreq);
3059 return ret;
3060 }
a2efcfa0 3061
1da177e4 3062 case PACKET_RX_RING:
69e3c75f 3063 case PACKET_TX_RING:
1da177e4 3064 {
f6fb8f10 3065 union tpacket_req_u req_u;
3066 int len;
1da177e4 3067
f6fb8f10 3068 switch (po->tp_version) {
3069 case TPACKET_V1:
3070 case TPACKET_V2:
3071 len = sizeof(req_u.req);
3072 break;
3073 case TPACKET_V3:
3074 default:
3075 len = sizeof(req_u.req3);
3076 break;
3077 }
3078 if (optlen < len)
1da177e4 3079 return -EINVAL;
bfd5f4a3
SS		 3080 if (pkt_sk(sk)->has_vnet_hdr)
3081 return -EINVAL;
f6fb8f10 3082 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3083 return -EFAULT;
f6fb8f10 3084 return packet_set_ring(sk, &req_u, 0,
3085 optname == PACKET_TX_RING);
1da177e4
LT		 3086 }
3087 case PACKET_COPY_THRESH:
3088 {
3089 int val;
3090
40d4e3df 3091 if (optlen != sizeof(val))
1da177e4 3092 return -EINVAL;
40d4e3df 3093 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3094 return -EFAULT;
3095
3096 pkt_sk(sk)->copy_thresh = val;
3097 return 0;
3098 }
bbd6ef87
PM		 3099 case PACKET_VERSION:
3100 {
3101 int val;
3102
3103 if (optlen != sizeof(val))
3104 return -EINVAL;
69e3c75f 3105 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3106 return -EBUSY;
3107 if (copy_from_user(&val, optval, sizeof(val)))
3108 return -EFAULT;
3109 switch (val) {
3110 case TPACKET_V1:
3111 case TPACKET_V2:
f6fb8f10 3112 case TPACKET_V3:
bbd6ef87
PM		 3113 po->tp_version = val;
3114 return 0;
3115 default:
3116 return -EINVAL;
3117 }
3118 }
8913336a
PM		 3119 case PACKET_RESERVE:
3120 {
3121 unsigned int val;
3122
3123 if (optlen != sizeof(val))
3124 return -EINVAL;
69e3c75f 3125 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3126 return -EBUSY;
3127 if (copy_from_user(&val, optval, sizeof(val)))
3128 return -EFAULT;
3129 po->tp_reserve = val;
3130 return 0;
3131 }
69e3c75f
JB		 3132 case PACKET_LOSS:
3133 {
3134 unsigned int val;
3135
3136 if (optlen != sizeof(val))
3137 return -EINVAL;
3138 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3139 return -EBUSY;
3140 if (copy_from_user(&val, optval, sizeof(val)))
3141 return -EFAULT;
3142 po->tp_loss = !!val;
3143 return 0;
3144 }
8dc41944
HX		 3145 case PACKET_AUXDATA:
3146 {
3147 int val;
3148
3149 if (optlen < sizeof(val))
3150 return -EINVAL;
3151 if (copy_from_user(&val, optval, sizeof(val)))
3152 return -EFAULT;
3153
3154 po->auxdata = !!val;
3155 return 0;
3156 }
80feaacb
PWJ		 3157 case PACKET_ORIGDEV:
3158 {
3159 int val;
3160
3161 if (optlen < sizeof(val))
3162 return -EINVAL;
3163 if (copy_from_user(&val, optval, sizeof(val)))
3164 return -EFAULT;
3165
3166 po->origdev = !!val;
3167 return 0;
3168 }
bfd5f4a3
SS		 3169 case PACKET_VNET_HDR:
3170 {
3171 int val;
3172
3173 if (sock->type != SOCK_RAW)
3174 return -EINVAL;
3175 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3176 return -EBUSY;
3177 if (optlen < sizeof(val))
3178 return -EINVAL;
3179 if (copy_from_user(&val, optval, sizeof(val)))
3180 return -EFAULT;
3181
3182 po->has_vnet_hdr = !!val;
3183 return 0;
3184 }
614f60fa
SM		 3185 case PACKET_TIMESTAMP:
3186 {
3187 int val;
3188
3189 if (optlen != sizeof(val))
3190 return -EINVAL;
3191 if (copy_from_user(&val, optval, sizeof(val)))
3192 return -EFAULT;
3193
3194 po->tp_tstamp = val;
3195 return 0;
3196 }
dc99f600
DM		 3197 case PACKET_FANOUT:
3198 {
3199 int val;
3200
3201 if (optlen != sizeof(val))
3202 return -EINVAL;
3203 if (copy_from_user(&val, optval, sizeof(val)))
3204 return -EFAULT;
3205
3206 return fanout_add(sk, val & 0xffff, val >> 16);
3207 }
5920cd3a
PC		 3208 case PACKET_TX_HAS_OFF:
3209 {
3210 unsigned int val;
3211
3212 if (optlen != sizeof(val))
3213 return -EINVAL;
3214 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3215 return -EBUSY;
3216 if (copy_from_user(&val, optval, sizeof(val)))
3217 return -EFAULT;
3218 po->tp_tx_has_off = !!val;
3219 return 0;
3220 }
1da177e4
LT		 3221 default:
3222 return -ENOPROTOOPT;
3223 }
3224}
3225
3226static int packet_getsockopt(struct socket *sock, int level, int optname,
3227 char __user *optval, int __user *optlen)
3228{
3229 int len;
c06fff6e 3230 int val, lv = sizeof(val);
1da177e4
LT		 3231 struct sock *sk = sock->sk;
3232 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3233 void *data = &val;
ee80fbf3 3234 union tpacket_stats_u st;
1da177e4
LT		 3235
3236 if (level != SOL_PACKET)
3237 return -ENOPROTOOPT;
3238
8ae55f04
KK		 3239 if (get_user(len, optlen))
3240 return -EFAULT;
1da177e4
LT		 3241
3242 if (len < 0)
3243 return -EINVAL;
1ce4f28b 3244
69e3c75f 3245 switch (optname) {
1da177e4 3246 case PACKET_STATISTICS:
1da177e4 3247 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3248 memcpy(&st, &po->stats, sizeof(st));
3249 memset(&po->stats, 0, sizeof(po->stats));
3250 spin_unlock_bh(&sk->sk_receive_queue.lock);
3251
f6fb8f10 3252 if (po->tp_version == TPACKET_V3) {
c06fff6e 3253 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3254 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3255 data = &st.stats3;
f6fb8f10 3256 } else {
c06fff6e 3257 lv = sizeof(struct tpacket_stats);
8bcdeaff 3258 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3259 data = &st.stats1;
f6fb8f10 3260 }
ee80fbf3 3261
8dc41944
HX		 3262 break;
3263 case PACKET_AUXDATA:
8dc41944 3264 val = po->auxdata;
80feaacb
PWJ		 3265 break;
3266 case PACKET_ORIGDEV:
80feaacb 3267 val = po->origdev;
bfd5f4a3
SS		 3268 break;
3269 case PACKET_VNET_HDR:
bfd5f4a3 3270 val = po->has_vnet_hdr;
1da177e4 3271 break;
bbd6ef87 3272 case PACKET_VERSION:
bbd6ef87 3273 val = po->tp_version;
bbd6ef87
PM		 3274 break;
3275 case PACKET_HDRLEN:
3276 if (len > sizeof(int))
3277 len = sizeof(int);
3278 if (copy_from_user(&val, optval, len))
3279 return -EFAULT;
3280 switch (val) {
3281 case TPACKET_V1:
3282 val = sizeof(struct tpacket_hdr);
3283 break;
3284 case TPACKET_V2:
3285 val = sizeof(struct tpacket2_hdr);
3286 break;
f6fb8f10 3287 case TPACKET_V3:
3288 val = sizeof(struct tpacket3_hdr);
3289 break;
bbd6ef87
PM		 3290 default:
3291 return -EINVAL;
3292 }
bbd6ef87 3293 break;
8913336a 3294 case PACKET_RESERVE:
8913336a 3295 val = po->tp_reserve;
8913336a 3296 break;
69e3c75f 3297 case PACKET_LOSS:
69e3c75f 3298 val = po->tp_loss;
69e3c75f 3299 break;
614f60fa 3300 case PACKET_TIMESTAMP:
614f60fa 3301 val = po->tp_tstamp;
614f60fa 3302 break;
dc99f600 3303 case PACKET_FANOUT:
dc99f600
DM		 3304 val = (po->fanout ?
3305 ((u32)po->fanout->id |
77f65ebd
WB		 3306 ((u32)po->fanout->type << 16) |
3307 ((u32)po->fanout->flags << 24)) :
dc99f600 3308 0);
dc99f600 3309 break;
5920cd3a
PC		 3310 case PACKET_TX_HAS_OFF:
3311 val = po->tp_tx_has_off;
3312 break;
1da177e4
LT		 3313 default:
3314 return -ENOPROTOOPT;
3315 }
3316
c06fff6e
ED		 3317 if (len > lv)
3318 len = lv;
8ae55f04
KK		 3319 if (put_user(len, optlen))
3320 return -EFAULT;
8dc41944
HX		 3321 if (copy_to_user(optval, data, len))
3322 return -EFAULT;
8ae55f04 3323 return 0;
1da177e4
LT		 3324}
3325
3326
351638e7
JP		 3327static int packet_notifier(struct notifier_block *this,
3328 unsigned long msg, void *ptr)
1da177e4
LT		 3329{
3330 struct sock *sk;
351638e7 3331 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3332 struct net *net = dev_net(dev);
1da177e4 3333
808f5114 3334 rcu_read_lock();
b67bfe0d 3335 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3336 struct packet_sock *po = pkt_sk(sk);
3337
3338 switch (msg) {
3339 case NETDEV_UNREGISTER:
1da177e4
LT		 3340 if (po->mclist)
3341 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3342 /* fallthrough */
3343
1da177e4
LT		 3344 case NETDEV_DOWN:
3345 if (dev->ifindex == po->ifindex) {
3346 spin_lock(&po->bind_lock);
3347 if (po->running) {
ce06b03e 3348 __unregister_prot_hook(sk, false);
1da177e4
LT		 3349 sk->sk_err = ENETDOWN;
3350 if (!sock_flag(sk, SOCK_DEAD))
3351 sk->sk_error_report(sk);
3352 }
3353 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3354 packet_cached_dev_reset(po);
1da177e4 3355 po->ifindex = -1;
160ff18a
BG		 3356 if (po->prot_hook.dev)
3357 dev_put(po->prot_hook.dev);
1da177e4
LT		 3358 po->prot_hook.dev = NULL;
3359 }
3360 spin_unlock(&po->bind_lock);
3361 }
3362 break;
3363 case NETDEV_UP:
808f5114 3364 if (dev->ifindex == po->ifindex) {
3365 spin_lock(&po->bind_lock);
ce06b03e
DM		 3366 if (po->num)
3367 register_prot_hook(sk);
808f5114 3368 spin_unlock(&po->bind_lock);
1da177e4 3369 }
1da177e4
LT		 3370 break;
3371 }
3372 }
808f5114 3373 rcu_read_unlock();
1da177e4
LT		 3374 return NOTIFY_DONE;
3375}
3376
3377
3378static int packet_ioctl(struct socket *sock, unsigned int cmd,
3379 unsigned long arg)
3380{
3381 struct sock *sk = sock->sk;
3382
69e3c75f 3383 switch (cmd) {
40d4e3df
ED		 3384 case SIOCOUTQ:
3385 {
3386 int amount = sk_wmem_alloc_get(sk);
31e6d363 3387
40d4e3df
ED		 3388 return put_user(amount, (int __user *)arg);
3389 }
3390 case SIOCINQ:
3391 {
3392 struct sk_buff *skb;
3393 int amount = 0;
3394
3395 spin_lock_bh(&sk->sk_receive_queue.lock);
3396 skb = skb_peek(&sk->sk_receive_queue);
3397 if (skb)
3398 amount = skb->len;
3399 spin_unlock_bh(&sk->sk_receive_queue.lock);
3400 return put_user(amount, (int __user *)arg);
3401 }
3402 case SIOCGSTAMP:
3403 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3404 case SIOCGSTAMPNS:
3405 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3406
1da177e4 3407#ifdef CONFIG_INET
40d4e3df
ED		 3408 case SIOCADDRT:
3409 case SIOCDELRT:
3410 case SIOCDARP:
3411 case SIOCGARP:
3412 case SIOCSARP:
3413 case SIOCGIFADDR:
3414 case SIOCSIFADDR:
3415 case SIOCGIFBRDADDR:
3416 case SIOCSIFBRDADDR:
3417 case SIOCGIFNETMASK:
3418 case SIOCSIFNETMASK:
3419 case SIOCGIFDSTADDR:
3420 case SIOCSIFDSTADDR:
3421 case SIOCSIFFLAGS:
40d4e3df 3422 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3423#endif
3424
40d4e3df
ED		 3425 default:
3426 return -ENOIOCTLCMD;
1da177e4
LT		 3427 }
3428 return 0;
3429}
3430
40d4e3df 3431static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3432 poll_table *wait)
3433{
3434 struct sock *sk = sock->sk;
3435 struct packet_sock *po = pkt_sk(sk);
3436 unsigned int mask = datagram_poll(file, sock, wait);
3437
3438 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3439 if (po->rx_ring.pg_vec) {
f6fb8f10 3440 if (!packet_previous_rx_frame(po, &po->rx_ring,
3441 TP_STATUS_KERNEL))
1da177e4
LT		 3442 mask |= POLLIN | POLLRDNORM;
3443 }
3444 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3445 spin_lock_bh(&sk->sk_write_queue.lock);
3446 if (po->tx_ring.pg_vec) {
3447 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3448 mask |= POLLOUT | POLLWRNORM;
3449 }
3450 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3451 return mask;
3452}
3453
3454
3455/* Dirty? Well, I still did not learn better way to account
3456 * for user mmaps.
3457 */
3458
3459static void packet_mm_open(struct vm_area_struct *vma)
3460{
3461 struct file *file = vma->vm_file;
40d4e3df 3462 struct socket *sock = file->private_data;
1da177e4 3463 struct sock *sk = sock->sk;
1ce4f28b 3464
1da177e4
LT		 3465 if (sk)
3466 atomic_inc(&pkt_sk(sk)->mapped);
3467}
3468
3469static void packet_mm_close(struct vm_area_struct *vma)
3470{
3471 struct file *file = vma->vm_file;
40d4e3df 3472 struct socket *sock = file->private_data;
1da177e4 3473 struct sock *sk = sock->sk;
1ce4f28b 3474
1da177e4
LT		 3475 if (sk)
3476 atomic_dec(&pkt_sk(sk)->mapped);
3477}
3478
f0f37e2f 3479static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3480 .open = packet_mm_open,
3481 .close = packet_mm_close,
1da177e4
LT		 3482};
3483
0e3125c7
NH		 3484static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3485 unsigned int len)
1da177e4
LT		 3486{
3487 int i;
3488
4ebf0ae2 3489 for (i = 0; i < len; i++) {
0e3125c7 3490 if (likely(pg_vec[i].buffer)) {
c56b4d90 3491 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3492 vfree(pg_vec[i].buffer);
3493 else
3494 free_pages((unsigned long)pg_vec[i].buffer,
3495 order);
3496 pg_vec[i].buffer = NULL;
3497 }
1da177e4
LT		 3498 }
3499 kfree(pg_vec);
3500}
3501
eea49cc9 3502static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3503{
0e3125c7
NH		 3504 char *buffer = NULL;
3505 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3506 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3507
3508 buffer = (char *) __get_free_pages(gfp_flags, order);
3509
3510 if (buffer)
3511 return buffer;
3512
3513 /*
3514 * __get_free_pages failed, fall back to vmalloc
3515 */
bbce5a59 3516 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3517
0e3125c7
NH		 3518 if (buffer)
3519 return buffer;
3520
3521 /*
3522 * vmalloc failed, lets dig into swap here
3523 */
0e3125c7
NH		 3524 gfp_flags &= ~__GFP_NORETRY;
3525 buffer = (char *)__get_free_pages(gfp_flags, order);
3526 if (buffer)
3527 return buffer;
3528
3529 /*
3530 * complete and utter failure
3531 */
3532 return NULL;
4ebf0ae2
DM		 3533}
3534
0e3125c7 3535static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3536{
3537 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3538 struct pgv *pg_vec;
4ebf0ae2
DM		 3539 int i;
3540
0e3125c7 3541 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3542 if (unlikely(!pg_vec))
3543 goto out;
3544
3545 for (i = 0; i < block_nr; i++) {
c56b4d90 3546 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3547 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3548 goto out_free_pgvec;
3549 }
3550
3551out:
3552 return pg_vec;
3553
3554out_free_pgvec:
3555 free_pg_vec(pg_vec, order, block_nr);
3556 pg_vec = NULL;
3557 goto out;
3558}
1da177e4 3559
f6fb8f10 3560static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3561 int closing, int tx_ring)
1da177e4 3562{
0e3125c7 3563 struct pgv *pg_vec = NULL;
1da177e4 3564 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3565 int was_running, order = 0;
69e3c75f
JB		 3566 struct packet_ring_buffer *rb;
3567 struct sk_buff_head *rb_queue;
0e11c91e 3568 __be16 num;
f6fb8f10 3569 int err = -EINVAL;
3570 /* Added to avoid minimal code churn */
3571 struct tpacket_req *req = &req_u->req;
3572
3573 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3574 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3575 WARN(1, "Tx-ring is not supported.\n");
3576 goto out;
3577 }
1ce4f28b 3578
69e3c75f
JB		 3579 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3580 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3581
69e3c75f
JB		 3582 err = -EBUSY;
3583 if (!closing) {
3584 if (atomic_read(&po->mapped))
3585 goto out;
3586 if (atomic_read(&rb->pending))
3587 goto out;
3588 }
1da177e4 3589
69e3c75f
JB		 3590 if (req->tp_block_nr) {
3591 /* Sanity tests and some calculations */
3592 err = -EBUSY;
3593 if (unlikely(rb->pg_vec))
3594 goto out;
1da177e4 3595
bbd6ef87
PM		 3596 switch (po->tp_version) {
3597 case TPACKET_V1:
3598 po->tp_hdrlen = TPACKET_HDRLEN;
3599 break;
3600 case TPACKET_V2:
3601 po->tp_hdrlen = TPACKET2_HDRLEN;
3602 break;
f6fb8f10 3603 case TPACKET_V3:
3604 po->tp_hdrlen = TPACKET3_HDRLEN;
3605 break;
bbd6ef87
PM		 3606 }
3607
69e3c75f 3608 err = -EINVAL;
4ebf0ae2 3609 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3610 goto out;
4ebf0ae2 3611 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3612 goto out;
8913336a 3613 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3614 po->tp_reserve))
3615 goto out;
4ebf0ae2 3616 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3617 goto out;
1da177e4 3618
69e3c75f
JB		 3619 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3620 if (unlikely(rb->frames_per_block <= 0))
3621 goto out;
3622 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3623 req->tp_frame_nr))
3624 goto out;
1da177e4
LT		 3625
3626 err = -ENOMEM;
4ebf0ae2
DM		 3627 order = get_order(req->tp_block_size);
3628 pg_vec = alloc_pg_vec(req, order);
3629 if (unlikely(!pg_vec))
1da177e4 3630 goto out;
f6fb8f10 3631 switch (po->tp_version) {
3632 case TPACKET_V3:
3633 /* Transmit path is not supported. We checked
3634 * it above but just being paranoid
3635 */
3636 if (!tx_ring)
3637 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3638 break;
3639 default:
3640 break;
3641 }
69e3c75f
JB		 3642 }
3643 /* Done */
3644 else {
3645 err = -EINVAL;
4ebf0ae2 3646 if (unlikely(req->tp_frame_nr))
69e3c75f 3647 goto out;
1da177e4
LT		 3648 }
3649
3650 lock_sock(sk);
3651
3652 /* Detach socket from network */
3653 spin_lock(&po->bind_lock);
3654 was_running = po->running;
3655 num = po->num;
3656 if (was_running) {
1da177e4 3657 po->num = 0;
ce06b03e 3658 __unregister_prot_hook(sk, false);
1da177e4
LT		 3659 }
3660 spin_unlock(&po->bind_lock);
1ce4f28b 3661
1da177e4
LT		 3662 synchronize_net();
3663
3664 err = -EBUSY;
905db440 3665 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3666 if (closing || atomic_read(&po->mapped) == 0) {
3667 err = 0;
69e3c75f 3668 spin_lock_bh(&rb_queue->lock);
c053fd96 3669 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3670 rb->frame_max = (req->tp_frame_nr - 1);
3671 rb->head = 0;
3672 rb->frame_size = req->tp_frame_size;
3673 spin_unlock_bh(&rb_queue->lock);
3674
c053fd96
CG		 3675 swap(rb->pg_vec_order, order);
3676 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3677
3678 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3679 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3680 tpacket_rcv : packet_rcv;
3681 skb_queue_purge(rb_queue);
1da177e4 3682 if (atomic_read(&po->mapped))
40d4e3df
ED		 3683 pr_err("packet_mmap: vma is busy: %d\n",
3684 atomic_read(&po->mapped));
1da177e4 3685 }
905db440 3686 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3687
3688 spin_lock(&po->bind_lock);
ce06b03e 3689 if (was_running) {
1da177e4 3690 po->num = num;
ce06b03e 3691 register_prot_hook(sk);
1da177e4
LT		 3692 }
3693 spin_unlock(&po->bind_lock);
f6fb8f10 3694 if (closing && (po->tp_version > TPACKET_V2)) {
3695 /* Because we don't support block-based V3 on tx-ring */
3696 if (!tx_ring)
3697 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3698 }
1da177e4
LT		 3699 release_sock(sk);
3700
1da177e4
LT		 3701 if (pg_vec)
3702 free_pg_vec(pg_vec, order, req->tp_block_nr);
3703out:
3704 return err;
3705}
3706
69e3c75f
JB		 3707static int packet_mmap(struct file *file, struct socket *sock,
3708 struct vm_area_struct *vma)
1da177e4
LT		 3709{
3710 struct sock *sk = sock->sk;
3711 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3712 unsigned long size, expected_size;
3713 struct packet_ring_buffer *rb;
1da177e4
LT		 3714 unsigned long start;
3715 int err = -EINVAL;
3716 int i;
3717
3718 if (vma->vm_pgoff)
3719 return -EINVAL;
3720
905db440 3721 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3722
3723 expected_size = 0;
3724 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3725 if (rb->pg_vec) {
3726 expected_size += rb->pg_vec_len
3727 * rb->pg_vec_pages
3728 * PAGE_SIZE;
3729 }
3730 }
3731
3732 if (expected_size == 0)
1da177e4 3733 goto out;
69e3c75f
JB		 3734
3735 size = vma->vm_end - vma->vm_start;
3736 if (size != expected_size)
1da177e4
LT		 3737 goto out;
3738
1da177e4 3739 start = vma->vm_start;
69e3c75f
JB		 3740 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3741 if (rb->pg_vec == NULL)
3742 continue;
3743
3744 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3745 struct page *page;
3746 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3747 int pg_num;
3748
c56b4d90
CG		 3749 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3750 page = pgv_to_page(kaddr);
69e3c75f
JB		 3751 err = vm_insert_page(vma, start, page);
3752 if (unlikely(err))
3753 goto out;
3754 start += PAGE_SIZE;
0e3125c7 3755 kaddr += PAGE_SIZE;
69e3c75f 3756 }
4ebf0ae2 3757 }
1da177e4 3758 }
69e3c75f 3759
4ebf0ae2 3760 atomic_inc(&po->mapped);
1da177e4
LT		 3761 vma->vm_ops = &packet_mmap_ops;
3762 err = 0;
3763
3764out:
905db440 3765 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3766 return err;
3767}
1da177e4 3768
90ddc4f0 3769static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3770 .family = PF_PACKET,
3771 .owner = THIS_MODULE,
3772 .release = packet_release,
3773 .bind = packet_bind_spkt,
3774 .connect = sock_no_connect,
3775 .socketpair = sock_no_socketpair,
3776 .accept = sock_no_accept,
3777 .getname = packet_getname_spkt,
3778 .poll = datagram_poll,
3779 .ioctl = packet_ioctl,
3780 .listen = sock_no_listen,
3781 .shutdown = sock_no_shutdown,
3782 .setsockopt = sock_no_setsockopt,
3783 .getsockopt = sock_no_getsockopt,
3784 .sendmsg = packet_sendmsg_spkt,
3785 .recvmsg = packet_recvmsg,
3786 .mmap = sock_no_mmap,
3787 .sendpage = sock_no_sendpage,
3788};
1da177e4 3789
90ddc4f0 3790static const struct proto_ops packet_ops = {
1da177e4
LT		 3791 .family = PF_PACKET,
3792 .owner = THIS_MODULE,
3793 .release = packet_release,
3794 .bind = packet_bind,
3795 .connect = sock_no_connect,
3796 .socketpair = sock_no_socketpair,
3797 .accept = sock_no_accept,
1ce4f28b 3798 .getname = packet_getname,
1da177e4
LT		 3799 .poll = packet_poll,
3800 .ioctl = packet_ioctl,
3801 .listen = sock_no_listen,
3802 .shutdown = sock_no_shutdown,
3803 .setsockopt = packet_setsockopt,
3804 .getsockopt = packet_getsockopt,
3805 .sendmsg = packet_sendmsg,
3806 .recvmsg = packet_recvmsg,
3807 .mmap = packet_mmap,
3808 .sendpage = sock_no_sendpage,
3809};
3810
ec1b4cf7 3811static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3812 .family = PF_PACKET,
3813 .create = packet_create,
3814 .owner = THIS_MODULE,
3815};
3816
3817static struct notifier_block packet_netdev_notifier = {
40d4e3df 3818 .notifier_call = packet_notifier,
1da177e4
LT		 3819};
3820
3821#ifdef CONFIG_PROC_FS
1da177e4
LT		 3822
3823static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3824 __acquires(RCU)
1da177e4 3825{
e372c414 3826 struct net *net = seq_file_net(seq);
808f5114 3827
3828 rcu_read_lock();
3829 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3830}
3831
3832static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3833{
1bf40954 3834 struct net *net = seq_file_net(seq);
808f5114 3835 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3836}
3837
3838static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3839 __releases(RCU)
1da177e4 3840{
808f5114 3841 rcu_read_unlock();
1da177e4
LT		 3842}
3843
1ce4f28b 3844static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3845{
3846 if (v == SEQ_START_TOKEN)
3847 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3848 else {
b7ceabd9 3849 struct sock *s = sk_entry(v);
1da177e4
LT		 3850 const struct packet_sock *po = pkt_sk(s);
3851
3852 seq_printf(seq,
71338aa7 3853 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3854 s,
3855 atomic_read(&s->sk_refcnt),
3856 s->sk_type,
3857 ntohs(po->num),
3858 po->ifindex,
3859 po->running,
3860 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3861 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3862 sock_i_ino(s));
1da177e4
LT		 3863 }
3864
3865 return 0;
3866}
3867
56b3d975 3868static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3869 .start = packet_seq_start,
3870 .next = packet_seq_next,
3871 .stop = packet_seq_stop,
3872 .show = packet_seq_show,
3873};
3874
3875static int packet_seq_open(struct inode *inode, struct file *file)
3876{
e372c414
DL		 3877 return seq_open_net(inode, file, &packet_seq_ops,
3878 sizeof(struct seq_net_private));
1da177e4
LT		 3879}
3880
da7071d7 3881static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3882 .owner = THIS_MODULE,
3883 .open = packet_seq_open,
3884 .read = seq_read,
3885 .llseek = seq_lseek,
e372c414 3886 .release = seq_release_net,
1da177e4
LT		 3887};
3888
3889#endif
3890
2c8c1e72 3891static int __net_init packet_net_init(struct net *net)
d12d01d6 3892{
0fa7fa98 3893 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3894 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3895
d4beaa66 3896 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3897 return -ENOMEM;
3898
3899 return 0;
3900}
3901
2c8c1e72 3902static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3903{
ece31ffd 3904 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3905}
3906
3907static struct pernet_operations packet_net_ops = {
3908 .init = packet_net_init,
3909 .exit = packet_net_exit,
3910};
3911
3912
1da177e4
LT		 3913static void __exit packet_exit(void)
3914{
1da177e4 3915 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3916 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3917 sock_unregister(PF_PACKET);
3918 proto_unregister(&packet_proto);
3919}
3920
3921static int __init packet_init(void)
3922{
3923 int rc = proto_register(&packet_proto, 0);
3924
3925 if (rc != 0)
3926 goto out;
3927
3928 sock_register(&packet_family_ops);
d12d01d6 3929 register_pernet_subsys(&packet_net_ops);
1da177e4 3930 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3931out:
3932 return rc;
3933}
3934
3935module_init(packet_init);
3936module_exit(packet_exit);
3937MODULE_LICENSE("GPL");
3938MODULE_ALIAS_NETPROTO(PF_PACKET);
Ubuntu Jammy Linux kernel mirror