]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - net/packet/af_packet.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: fec: reduce spin lock time in fec_ptp_adjfreq
[mirror_ubuntu-zesty-kernel.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
1da177e4
LT		 91
92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
f6fb8f10 161static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 162 int closing, int tx_ring);
163
f6fb8f10 164
165#define V3_ALIGNMENT (8)
166
bc59ba39 167#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 168
169#define BLK_PLUS_PRIV(sz_of_priv) \
170 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
171
f6fb8f10 172#define PGV_FROM_VMALLOC 1
69e3c75f 173
f6fb8f10 174#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
175#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
176#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
177#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
178#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
179#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
180#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
181
69e3c75f
JB		 182struct packet_sock;
183static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
1da177e4 184
f6fb8f10 185static void *packet_previous_frame(struct packet_sock *po,
186 struct packet_ring_buffer *rb,
187 int status);
188static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 189static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
190 struct tpacket_block_desc *);
191static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 192 struct packet_sock *);
bc59ba39 193static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 194 struct packet_sock *, unsigned int status);
bc59ba39 195static int prb_queue_frozen(struct tpacket_kbdq_core *);
196static void prb_open_block(struct tpacket_kbdq_core *,
197 struct tpacket_block_desc *);
f6fb8f10 198static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 199static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
200static void prb_init_blk_timer(struct packet_sock *,
201 struct tpacket_kbdq_core *,
202 void (*func) (unsigned long));
203static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
204static void prb_clear_rxhash(struct tpacket_kbdq_core *,
205 struct tpacket3_hdr *);
206static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
207 struct tpacket3_hdr *);
1da177e4
LT		 208static void packet_flush_mclist(struct sock *sk);
209
ffbc6111
HX		 210struct packet_skb_cb {
211 unsigned int origlen;
212 union {
213 struct sockaddr_pkt pkt;
214 struct sockaddr_ll ll;
215 } sa;
216};
217
218#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 219
bc59ba39 220#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 221#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 222 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 223#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 224 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 225#define GET_NEXT_PRB_BLK_NUM(x) \
226 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
227 ((x)->kactive_blk_num+1) : 0)
228
dc99f600
DM		 229static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
230static void __fanout_link(struct sock *sk, struct packet_sock *po);
231
ce06b03e
DM		 232/* register_prot_hook must be invoked with the po->bind_lock held,
233 * or from a context in which asynchronous accesses to the packet
234 * socket is not possible (packet_create()).
235 */
236static void register_prot_hook(struct sock *sk)
237{
238 struct packet_sock *po = pkt_sk(sk);
239 if (!po->running) {
dc99f600
DM		 240 if (po->fanout)
241 __fanout_link(sk, po);
242 else
243 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 244 sock_hold(sk);
245 po->running = 1;
246 }
247}
248
249/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
250 * held. If the sync parameter is true, we will temporarily drop
251 * the po->bind_lock and do a synchronize_net to make sure no
252 * asynchronous packet processing paths still refer to the elements
253 * of po->prot_hook. If the sync parameter is false, it is the
254 * callers responsibility to take care of this.
255 */
256static void __unregister_prot_hook(struct sock *sk, bool sync)
257{
258 struct packet_sock *po = pkt_sk(sk);
259
260 po->running = 0;
dc99f600
DM		 261 if (po->fanout)
262 __fanout_unlink(sk, po);
263 else
264 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 265 __sock_put(sk);
266
267 if (sync) {
268 spin_unlock(&po->bind_lock);
269 synchronize_net();
270 spin_lock(&po->bind_lock);
271 }
272}
273
274static void unregister_prot_hook(struct sock *sk, bool sync)
275{
276 struct packet_sock *po = pkt_sk(sk);
277
278 if (po->running)
279 __unregister_prot_hook(sk, sync);
280}
281
f6dafa95 282static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 283{
284 if (is_vmalloc_addr(addr))
285 return vmalloc_to_page(addr);
286 return virt_to_page(addr);
287}
288
69e3c75f 289static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 290{
bbd6ef87
PM		 291 union {
292 struct tpacket_hdr *h1;
293 struct tpacket2_hdr *h2;
294 void *raw;
295 } h;
1da177e4 296
69e3c75f 297 h.raw = frame;
bbd6ef87
PM		 298 switch (po->tp_version) {
299 case TPACKET_V1:
69e3c75f 300 h.h1->tp_status = status;
0af55bb5 301 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 302 break;
303 case TPACKET_V2:
69e3c75f 304 h.h2->tp_status = status;
0af55bb5 305 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 306 break;
f6fb8f10 307 case TPACKET_V3:
69e3c75f 308 default:
f6fb8f10 309 WARN(1, "TPACKET version not supported.\n");
69e3c75f 310 BUG();
bbd6ef87 311 }
69e3c75f
JB		 312
313 smp_wmb();
bbd6ef87
PM		 314}
315
69e3c75f 316static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87
PM		 317{
318 union {
319 struct tpacket_hdr *h1;
320 struct tpacket2_hdr *h2;
321 void *raw;
322 } h;
323
69e3c75f
JB		 324 smp_rmb();
325
bbd6ef87
PM		 326 h.raw = frame;
327 switch (po->tp_version) {
328 case TPACKET_V1:
0af55bb5 329 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 330 return h.h1->tp_status;
bbd6ef87 331 case TPACKET_V2:
0af55bb5 332 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 333 return h.h2->tp_status;
f6fb8f10 334 case TPACKET_V3:
69e3c75f 335 default:
f6fb8f10 336 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 337 BUG();
338 return 0;
bbd6ef87 339 }
1da177e4 340}
69e3c75f
JB		 341
342static void *packet_lookup_frame(struct packet_sock *po,
343 struct packet_ring_buffer *rb,
344 unsigned int position,
345 int status)
346{
347 unsigned int pg_vec_pos, frame_offset;
348 union {
349 struct tpacket_hdr *h1;
350 struct tpacket2_hdr *h2;
351 void *raw;
352 } h;
353
354 pg_vec_pos = position / rb->frames_per_block;
355 frame_offset = position % rb->frames_per_block;
356
0e3125c7
NH		 357 h.raw = rb->pg_vec[pg_vec_pos].buffer +
358 (frame_offset * rb->frame_size);
69e3c75f
JB		 359
360 if (status != __packet_get_status(po, h.raw))
361 return NULL;
362
363 return h.raw;
364}
365
eea49cc9 366static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 367 struct packet_ring_buffer *rb,
368 int status)
369{
370 return packet_lookup_frame(po, rb, rb->head, status);
371}
372
bc59ba39 373static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 374{
375 del_timer_sync(&pkc->retire_blk_timer);
376}
377
378static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
379 int tx_ring,
380 struct sk_buff_head *rb_queue)
381{
bc59ba39 382 struct tpacket_kbdq_core *pkc;
f6fb8f10 383
384 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
385
386 spin_lock(&rb_queue->lock);
387 pkc->delete_blk_timer = 1;
388 spin_unlock(&rb_queue->lock);
389
390 prb_del_retire_blk_timer(pkc);
391}
392
393static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 394 struct tpacket_kbdq_core *pkc,
f6fb8f10 395 void (*func) (unsigned long))
396{
397 init_timer(&pkc->retire_blk_timer);
398 pkc->retire_blk_timer.data = (long)po;
399 pkc->retire_blk_timer.function = func;
400 pkc->retire_blk_timer.expires = jiffies;
401}
402
403static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
404{
bc59ba39 405 struct tpacket_kbdq_core *pkc;
f6fb8f10 406
407 if (tx_ring)
408 BUG();
409
410 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
411 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
412}
413
414static int prb_calc_retire_blk_tmo(struct packet_sock *po,
415 int blk_size_in_bytes)
416{
417 struct net_device *dev;
418 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 419 struct ethtool_cmd ecmd;
420 int err;
e440cf2c 421 u32 speed;
f6fb8f10 422
4bc71cb9
JP		 423 rtnl_lock();
424 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
425 if (unlikely(!dev)) {
426 rtnl_unlock();
f6fb8f10 427 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 428 }
429 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 430 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 431 rtnl_unlock();
432 if (!err) {
4bc71cb9
JP		 433 /*
434 * If the link speed is so slow you don't really
435 * need to worry about perf anyways
436 */
e440cf2c 437 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 438 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 439 } else {
440 msec = 1;
441 div = speed / 1000;
f6fb8f10 442 }
443 }
444
445 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
446
447 if (div)
448 mbits /= div;
449
450 tmo = mbits * msec;
451
452 if (div)
453 return tmo+1;
454 return tmo;
455}
456
bc59ba39 457static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 458 union tpacket_req_u *req_u)
459{
460 p1->feature_req_word = req_u->req3.tp_feature_req_word;
461}
462
463static void init_prb_bdqc(struct packet_sock *po,
464 struct packet_ring_buffer *rb,
465 struct pgv *pg_vec,
466 union tpacket_req_u *req_u, int tx_ring)
467{
bc59ba39 468 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
469 struct tpacket_block_desc *pbd;
f6fb8f10 470
471 memset(p1, 0x0, sizeof(*p1));
472
473 p1->knxt_seq_num = 1;
474 p1->pkbdq = pg_vec;
bc59ba39 475 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 476 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 477 p1->kblk_size = req_u->req3.tp_block_size;
478 p1->knum_blocks = req_u->req3.tp_block_nr;
479 p1->hdrlen = po->tp_hdrlen;
480 p1->version = po->tp_version;
481 p1->last_kactive_blk_num = 0;
482 po->stats_u.stats3.tp_freeze_q_cnt = 0;
483 if (req_u->req3.tp_retire_blk_tov)
484 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
485 else
486 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
487 req_u->req3.tp_block_size);
488 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
489 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
490
491 prb_init_ft_ops(p1, req_u);
492 prb_setup_retire_blk_timer(po, tx_ring);
493 prb_open_block(p1, pbd);
494}
495
496/* Do NOT update the last_blk_num first.
497 * Assumes sk_buff_head lock is held.
498 */
bc59ba39 499static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 500{
501 mod_timer(&pkc->retire_blk_timer,
502 jiffies + pkc->tov_in_jiffies);
503 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
504}
505
506/*
507 * Timer logic:
508 * 1) We refresh the timer only when we open a block.
509 * By doing this we don't waste cycles refreshing the timer
510 * on packet-by-packet basis.
511 *
512 * With a 1MB block-size, on a 1Gbps line, it will take
513 * i) ~8 ms to fill a block + ii) memcpy etc.
514 * In this cut we are not accounting for the memcpy time.
515 *
516 * So, if the user sets the 'tmo' to 10ms then the timer
517 * will never fire while the block is still getting filled
518 * (which is what we want). However, the user could choose
519 * to close a block early and that's fine.
520 *
521 * But when the timer does fire, we check whether or not to refresh it.
522 * Since the tmo granularity is in msecs, it is not too expensive
523 * to refresh the timer, lets say every '8' msecs.
524 * Either the user can set the 'tmo' or we can derive it based on
525 * a) line-speed and b) block-size.
526 * prb_calc_retire_blk_tmo() calculates the tmo.
527 *
528 */
529static void prb_retire_rx_blk_timer_expired(unsigned long data)
530{
531 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 532 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 533 unsigned int frozen;
bc59ba39 534 struct tpacket_block_desc *pbd;
f6fb8f10 535
536 spin_lock(&po->sk.sk_receive_queue.lock);
537
538 frozen = prb_queue_frozen(pkc);
539 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
540
541 if (unlikely(pkc->delete_blk_timer))
542 goto out;
543
544 /* We only need to plug the race when the block is partially filled.
545 * tpacket_rcv:
546 * lock(); increment BLOCK_NUM_PKTS; unlock()
547 * copy_bits() is in progress ...
548 * timer fires on other cpu:
549 * we can't retire the current block because copy_bits
550 * is in progress.
551 *
552 */
553 if (BLOCK_NUM_PKTS(pbd)) {
554 while (atomic_read(&pkc->blk_fill_in_prog)) {
555 /* Waiting for skb_copy_bits to finish... */
556 cpu_relax();
557 }
558 }
559
560 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
561 if (!frozen) {
562 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
563 if (!prb_dispatch_next_block(pkc, po))
564 goto refresh_timer;
565 else
566 goto out;
567 } else {
568 /* Case 1. Queue was frozen because user-space was
569 * lagging behind.
570 */
571 if (prb_curr_blk_in_use(pkc, pbd)) {
572 /*
573 * Ok, user-space is still behind.
574 * So just refresh the timer.
575 */
576 goto refresh_timer;
577 } else {
578 /* Case 2. queue was frozen,user-space caught up,
579 * now the link went idle && the timer fired.
580 * We don't have a block to close.So we open this
581 * block and restart the timer.
582 * opening a block thaws the queue,restarts timer
583 * Thawing/timer-refresh is a side effect.
584 */
585 prb_open_block(pkc, pbd);
586 goto out;
587 }
588 }
589 }
590
591refresh_timer:
592 _prb_refresh_rx_retire_blk_timer(pkc);
593
594out:
595 spin_unlock(&po->sk.sk_receive_queue.lock);
596}
597
eea49cc9 598static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 599 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 600{
601 /* Flush everything minus the block header */
602
603#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
604 u8 *start, *end;
605
606 start = (u8 *)pbd1;
607
608 /* Skip the block header(we know header WILL fit in 4K) */
609 start += PAGE_SIZE;
610
611 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
612 for (; start < end; start += PAGE_SIZE)
613 flush_dcache_page(pgv_to_page(start));
614
615 smp_wmb();
616#endif
617
618 /* Now update the block status. */
619
620 BLOCK_STATUS(pbd1) = status;
621
622 /* Flush the block header */
623
624#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
625 start = (u8 *)pbd1;
626 flush_dcache_page(pgv_to_page(start));
627
628 smp_wmb();
629#endif
630}
631
632/*
633 * Side effect:
634 *
635 * 1) flush the block
636 * 2) Increment active_blk_num
637 *
638 * Note:We DONT refresh the timer on purpose.
639 * Because almost always the next block will be opened.
640 */
bc59ba39 641static void prb_close_block(struct tpacket_kbdq_core *pkc1,
642 struct tpacket_block_desc *pbd1,
f6fb8f10 643 struct packet_sock *po, unsigned int stat)
644{
645 __u32 status = TP_STATUS_USER | stat;
646
647 struct tpacket3_hdr *last_pkt;
bc59ba39 648 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 649
650 if (po->stats.tp_drops)
651 status |= TP_STATUS_LOSING;
652
653 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
654 last_pkt->tp_next_offset = 0;
655
656 /* Get the ts of the last pkt */
657 if (BLOCK_NUM_PKTS(pbd1)) {
658 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
659 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
660 } else {
661 /* Ok, we tmo'd - so get the current time */
662 struct timespec ts;
663 getnstimeofday(&ts);
664 h1->ts_last_pkt.ts_sec = ts.tv_sec;
665 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
666 }
667
668 smp_wmb();
669
670 /* Flush the block */
671 prb_flush_block(pkc1, pbd1, status);
672
673 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
674}
675
eea49cc9 676static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 677{
678 pkc->reset_pending_on_curr_blk = 0;
679}
680
681/*
682 * Side effect of opening a block:
683 *
684 * 1) prb_queue is thawed.
685 * 2) retire_blk_timer is refreshed.
686 *
687 */
bc59ba39 688static void prb_open_block(struct tpacket_kbdq_core *pkc1,
689 struct tpacket_block_desc *pbd1)
f6fb8f10 690{
691 struct timespec ts;
bc59ba39 692 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 693
694 smp_rmb();
695
696 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd1))) {
697
698 /* We could have just memset this but we will lose the
699 * flexibility of making the priv area sticky
700 */
701 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
702 BLOCK_NUM_PKTS(pbd1) = 0;
703 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
704 getnstimeofday(&ts);
705 h1->ts_first_pkt.ts_sec = ts.tv_sec;
706 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
707 pkc1->pkblk_start = (char *)pbd1;
e3192690 708 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 709 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
710 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
711 pbd1->version = pkc1->version;
712 pkc1->prev = pkc1->nxt_offset;
713 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
714 prb_thaw_queue(pkc1);
715 _prb_refresh_rx_retire_blk_timer(pkc1);
716
717 smp_wmb();
718
719 return;
720 }
721
722 WARN(1, "ERROR block:%p is NOT FREE status:%d kactive_blk_num:%d\n",
723 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num);
724 dump_stack();
725 BUG();
726}
727
728/*
729 * Queue freeze logic:
730 * 1) Assume tp_block_nr = 8 blocks.
731 * 2) At time 't0', user opens Rx ring.
732 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
733 * 4) user-space is either sleeping or processing block '0'.
734 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
735 * it will close block-7,loop around and try to fill block '0'.
736 * call-flow:
737 * __packet_lookup_frame_in_block
738 * prb_retire_current_block()
739 * prb_dispatch_next_block()
740 * |->(BLOCK_STATUS == USER) evaluates to true
741 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
742 * 6) Now there are two cases:
743 * 6.1) Link goes idle right after the queue is frozen.
744 * But remember, the last open_block() refreshed the timer.
745 * When this timer expires,it will refresh itself so that we can
746 * re-open block-0 in near future.
747 * 6.2) Link is busy and keeps on receiving packets. This is a simple
748 * case and __packet_lookup_frame_in_block will check if block-0
749 * is free and can now be re-used.
750 */
eea49cc9 751static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 752 struct packet_sock *po)
753{
754 pkc->reset_pending_on_curr_blk = 1;
755 po->stats_u.stats3.tp_freeze_q_cnt++;
756}
757
758#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
759
760/*
761 * If the next block is free then we will dispatch it
762 * and return a good offset.
763 * Else, we will freeze the queue.
764 * So, caller must check the return value.
765 */
bc59ba39 766static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 767 struct packet_sock *po)
768{
bc59ba39 769 struct tpacket_block_desc *pbd;
f6fb8f10 770
771 smp_rmb();
772
773 /* 1. Get current block num */
774 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
775
776 /* 2. If this block is currently in_use then freeze the queue */
777 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
778 prb_freeze_queue(pkc, po);
779 return NULL;
780 }
781
782 /*
783 * 3.
784 * open this block and return the offset where the first packet
785 * needs to get stored.
786 */
787 prb_open_block(pkc, pbd);
788 return (void *)pkc->nxt_offset;
789}
790
bc59ba39 791static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 792 struct packet_sock *po, unsigned int status)
793{
bc59ba39 794 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 795
796 /* retire/close the current block */
797 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
798 /*
799 * Plug the case where copy_bits() is in progress on
800 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
801 * have space to copy the pkt in the current block and
802 * called prb_retire_current_block()
803 *
804 * We don't need to worry about the TMO case because
805 * the timer-handler already handled this case.
806 */
807 if (!(status & TP_STATUS_BLK_TMO)) {
808 while (atomic_read(&pkc->blk_fill_in_prog)) {
809 /* Waiting for skb_copy_bits to finish... */
810 cpu_relax();
811 }
812 }
813 prb_close_block(pkc, pbd, po, status);
814 return;
815 }
816
817 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd);
818 dump_stack();
819 BUG();
820}
821
eea49cc9 822static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 823 struct tpacket_block_desc *pbd)
f6fb8f10 824{
825 return TP_STATUS_USER & BLOCK_STATUS(pbd);
826}
827
eea49cc9 828static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 829{
830 return pkc->reset_pending_on_curr_blk;
831}
832
eea49cc9 833static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 834{
bc59ba39 835 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 836 atomic_dec(&pkc->blk_fill_in_prog);
837}
838
eea49cc9 839static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 840 struct tpacket3_hdr *ppd)
841{
842 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
843}
844
eea49cc9 845static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 846 struct tpacket3_hdr *ppd)
847{
848 ppd->hv1.tp_rxhash = 0;
849}
850
eea49cc9 851static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 852 struct tpacket3_hdr *ppd)
853{
854 if (vlan_tx_tag_present(pkc->skb)) {
855 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
856 ppd->tp_status = TP_STATUS_VLAN_VALID;
857 } else {
9e67030a 858 ppd->hv1.tp_vlan_tci = 0;
859 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 860 }
861}
862
bc59ba39 863static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 864 struct tpacket3_hdr *ppd)
865{
866 prb_fill_vlan_info(pkc, ppd);
867
868 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
869 prb_fill_rxhash(pkc, ppd);
870 else
871 prb_clear_rxhash(pkc, ppd);
872}
873
eea49cc9 874static void prb_fill_curr_block(char *curr,
bc59ba39 875 struct tpacket_kbdq_core *pkc,
876 struct tpacket_block_desc *pbd,
f6fb8f10 877 unsigned int len)
878{
879 struct tpacket3_hdr *ppd;
880
881 ppd = (struct tpacket3_hdr *)curr;
882 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
883 pkc->prev = curr;
884 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
885 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
886 BLOCK_NUM_PKTS(pbd) += 1;
887 atomic_inc(&pkc->blk_fill_in_prog);
888 prb_run_all_ft_ops(pkc, ppd);
889}
890
891/* Assumes caller has the sk->rx_queue.lock */
892static void *__packet_lookup_frame_in_block(struct packet_sock *po,
893 struct sk_buff *skb,
894 int status,
895 unsigned int len
896 )
897{
bc59ba39 898 struct tpacket_kbdq_core *pkc;
899 struct tpacket_block_desc *pbd;
f6fb8f10 900 char *curr, *end;
901
e3192690 902 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 903 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
904
905 /* Queue is frozen when user space is lagging behind */
906 if (prb_queue_frozen(pkc)) {
907 /*
908 * Check if that last block which caused the queue to freeze,
909 * is still in_use by user-space.
910 */
911 if (prb_curr_blk_in_use(pkc, pbd)) {
912 /* Can't record this packet */
913 return NULL;
914 } else {
915 /*
916 * Ok, the block was released by user-space.
917 * Now let's open that block.
918 * opening a block also thaws the queue.
919 * Thawing is a side effect.
920 */
921 prb_open_block(pkc, pbd);
922 }
923 }
924
925 smp_mb();
926 curr = pkc->nxt_offset;
927 pkc->skb = skb;
e3192690 928 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 929
930 /* first try the current block */
931 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
932 prb_fill_curr_block(curr, pkc, pbd, len);
933 return (void *)curr;
934 }
935
936 /* Ok, close the current block */
937 prb_retire_current_block(pkc, po, 0);
938
939 /* Now, try to dispatch the next block */
940 curr = (char *)prb_dispatch_next_block(pkc, po);
941 if (curr) {
942 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
943 prb_fill_curr_block(curr, pkc, pbd, len);
944 return (void *)curr;
945 }
946
947 /*
948 * No free blocks are available.user_space hasn't caught up yet.
949 * Queue was just frozen and now this packet will get dropped.
950 */
951 return NULL;
952}
953
eea49cc9 954static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 955 struct sk_buff *skb,
956 int status, unsigned int len)
957{
958 char *curr = NULL;
959 switch (po->tp_version) {
960 case TPACKET_V1:
961 case TPACKET_V2:
962 curr = packet_lookup_frame(po, &po->rx_ring,
963 po->rx_ring.head, status);
964 return curr;
965 case TPACKET_V3:
966 return __packet_lookup_frame_in_block(po, skb, status, len);
967 default:
968 WARN(1, "TPACKET version not supported\n");
969 BUG();
99aa3473 970 return NULL;
f6fb8f10 971 }
972}
973
eea49cc9 974static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 975 struct packet_ring_buffer *rb,
976 unsigned int previous,
977 int status)
978{
bc59ba39 979 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
980 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, previous);
f6fb8f10 981
982 if (status != BLOCK_STATUS(pbd))
983 return NULL;
984 return pbd;
985}
986
eea49cc9 987static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 988{
989 unsigned int prev;
990 if (rb->prb_bdqc.kactive_blk_num)
991 prev = rb->prb_bdqc.kactive_blk_num-1;
992 else
993 prev = rb->prb_bdqc.knum_blocks-1;
994 return prev;
995}
996
997/* Assumes caller has held the rx_queue.lock */
eea49cc9 998static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 999 struct packet_ring_buffer *rb,
1000 int status)
1001{
1002 unsigned int previous = prb_previous_blk_num(rb);
1003 return prb_lookup_block(po, rb, previous, status);
1004}
1005
eea49cc9 1006static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1007 struct packet_ring_buffer *rb,
1008 int status)
1009{
1010 if (po->tp_version <= TPACKET_V2)
1011 return packet_previous_frame(po, rb, status);
1012
1013 return __prb_previous_block(po, rb, status);
1014}
1015
eea49cc9 1016static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1017 struct packet_ring_buffer *rb)
1018{
1019 switch (po->tp_version) {
1020 case TPACKET_V1:
1021 case TPACKET_V2:
1022 return packet_increment_head(rb);
1023 case TPACKET_V3:
1024 default:
1025 WARN(1, "TPACKET version not supported.\n");
1026 BUG();
1027 return;
1028 }
1029}
1030
eea49cc9 1031static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1032 struct packet_ring_buffer *rb,
1033 int status)
1034{
1035 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1036 return packet_lookup_frame(po, rb, previous, status);
1037}
1038
eea49cc9 1039static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1040{
1041 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1042}
1043
1da177e4
LT		 1044static void packet_sock_destruct(struct sock *sk)
1045{
ed85b565
RC		 1046 skb_queue_purge(&sk->sk_error_queue);
1047
547b792c
IJ		 1048 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1049 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1050
1051 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1052 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1053 return;
1054 }
1055
17ab56a2 1056 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1057}
1058
dc99f600
DM		 1059static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1060{
1061 int x = atomic_read(&f->rr_cur) + 1;
1062
1063 if (x >= num)
1064 x = 0;
1065
1066 return x;
1067}
1068
1069static struct sock *fanout_demux_hash(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1070{
1071 u32 idx, hash = skb->rxhash;
1072
1073 idx = ((u64)hash * num) >> 32;
1074
1075 return f->arr[idx];
1076}
1077
1078static struct sock *fanout_demux_lb(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1079{
1080 int cur, old;
1081
1082 cur = atomic_read(&f->rr_cur);
1083 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1084 fanout_rr_next(f, num))) != cur)
1085 cur = old;
1086 return f->arr[cur];
1087}
1088
95ec3eb4
DM		 1089static struct sock *fanout_demux_cpu(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1090{
1091 unsigned int cpu = smp_processor_id();
1092
1093 return f->arr[cpu % num];
1094}
1095
95ec3eb4
DM		 1096static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1097 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1098{
1099 struct packet_fanout *f = pt->af_packet_priv;
1100 unsigned int num = f->num_members;
1101 struct packet_sock *po;
1102 struct sock *sk;
1103
1104 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1105 !num) {
1106 kfree_skb(skb);
1107 return 0;
1108 }
1109
95ec3eb4
DM		 1110 switch (f->type) {
1111 case PACKET_FANOUT_HASH:
1112 default:
1113 if (f->defrag) {
bc416d97 1114 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1115 if (!skb)
1116 return 0;
1117 }
1118 skb_get_rxhash(skb);
1119 sk = fanout_demux_hash(f, skb, num);
1120 break;
1121 case PACKET_FANOUT_LB:
1122 sk = fanout_demux_lb(f, skb, num);
1123 break;
1124 case PACKET_FANOUT_CPU:
1125 sk = fanout_demux_cpu(f, skb, num);
1126 break;
dc99f600
DM		 1127 }
1128
dc99f600
DM		 1129 po = pkt_sk(sk);
1130
1131 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1132}
1133
fff3321d
PE		 1134DEFINE_MUTEX(fanout_mutex);
1135EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1136static LIST_HEAD(fanout_list);
1137
1138static void __fanout_link(struct sock *sk, struct packet_sock *po)
1139{
1140 struct packet_fanout *f = po->fanout;
1141
1142 spin_lock(&f->lock);
1143 f->arr[f->num_members] = sk;
1144 smp_wmb();
1145 f->num_members++;
1146 spin_unlock(&f->lock);
1147}
1148
1149static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1150{
1151 struct packet_fanout *f = po->fanout;
1152 int i;
1153
1154 spin_lock(&f->lock);
1155 for (i = 0; i < f->num_members; i++) {
1156 if (f->arr[i] == sk)
1157 break;
1158 }
1159 BUG_ON(i >= f->num_members);
1160 f->arr[i] = f->arr[f->num_members - 1];
1161 f->num_members--;
1162 spin_unlock(&f->lock);
1163}
1164
a0dfb263 1165static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1166{
1167 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1168 return true;
1169
1170 return false;
1171}
1172
7736d33f 1173static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1174{
1175 struct packet_sock *po = pkt_sk(sk);
1176 struct packet_fanout *f, *match;
7736d33f
DM		 1177 u8 type = type_flags & 0xff;
1178 u8 defrag = (type_flags & PACKET_FANOUT_FLAG_DEFRAG) ? 1 : 0;
dc99f600
DM		 1179 int err;
1180
1181 switch (type) {
1182 case PACKET_FANOUT_HASH:
1183 case PACKET_FANOUT_LB:
95ec3eb4 1184 case PACKET_FANOUT_CPU:
dc99f600
DM		 1185 break;
1186 default:
1187 return -EINVAL;
1188 }
1189
1190 if (!po->running)
1191 return -EINVAL;
1192
1193 if (po->fanout)
1194 return -EALREADY;
1195
1196 mutex_lock(&fanout_mutex);
1197 match = NULL;
1198 list_for_each_entry(f, &fanout_list, list) {
1199 if (f->id == id &&
1200 read_pnet(&f->net) == sock_net(sk)) {
1201 match = f;
1202 break;
1203 }
1204 }
afe62c68 1205 err = -EINVAL;
7736d33f 1206 if (match && match->defrag != defrag)
afe62c68 1207 goto out;
dc99f600 1208 if (!match) {
afe62c68 1209 err = -ENOMEM;
dc99f600 1210 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1211 if (!match)
1212 goto out;
1213 write_pnet(&match->net, sock_net(sk));
1214 match->id = id;
1215 match->type = type;
1216 match->defrag = defrag;
1217 atomic_set(&match->rr_cur, 0);
1218 INIT_LIST_HEAD(&match->list);
1219 spin_lock_init(&match->lock);
1220 atomic_set(&match->sk_ref, 0);
1221 match->prot_hook.type = po->prot_hook.type;
1222 match->prot_hook.dev = po->prot_hook.dev;
1223 match->prot_hook.func = packet_rcv_fanout;
1224 match->prot_hook.af_packet_priv = match;
c0de08d0 1225 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1226 dev_add_pack(&match->prot_hook);
1227 list_add(&match->list, &fanout_list);
dc99f600 1228 }
afe62c68
ED		 1229 err = -EINVAL;
1230 if (match->type == type &&
1231 match->prot_hook.type == po->prot_hook.type &&
1232 match->prot_hook.dev == po->prot_hook.dev) {
1233 err = -ENOSPC;
1234 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1235 __dev_remove_pack(&po->prot_hook);
1236 po->fanout = match;
1237 atomic_inc(&match->sk_ref);
1238 __fanout_link(sk, po);
1239 err = 0;
dc99f600
DM		 1240 }
1241 }
afe62c68 1242out:
dc99f600
DM		 1243 mutex_unlock(&fanout_mutex);
1244 return err;
1245}
1246
1247static void fanout_release(struct sock *sk)
1248{
1249 struct packet_sock *po = pkt_sk(sk);
1250 struct packet_fanout *f;
1251
1252 f = po->fanout;
1253 if (!f)
1254 return;
1255
fff3321d 1256 mutex_lock(&fanout_mutex);
dc99f600
DM		 1257 po->fanout = NULL;
1258
dc99f600
DM		 1259 if (atomic_dec_and_test(&f->sk_ref)) {
1260 list_del(&f->list);
1261 dev_remove_pack(&f->prot_hook);
1262 kfree(f);
1263 }
1264 mutex_unlock(&fanout_mutex);
1265}
1da177e4 1266
90ddc4f0 1267static const struct proto_ops packet_ops;
1da177e4 1268
90ddc4f0 1269static const struct proto_ops packet_ops_spkt;
1da177e4 1270
40d4e3df
ED		 1271static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1272 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1273{
1274 struct sock *sk;
1275 struct sockaddr_pkt *spkt;
1276
1277 /*
1278 * When we registered the protocol we saved the socket in the data
1279 * field for just this event.
1280 */
1281
1282 sk = pt->af_packet_priv;
1ce4f28b 1283
1da177e4
LT		 1284 /*
1285 * Yank back the headers [hope the device set this
1286 * right or kerboom...]
1287 *
1288 * Incoming packets have ll header pulled,
1289 * push it back.
1290 *
98e399f8 1291 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1292 * so that this procedure is noop.
1293 */
1294
1295 if (skb->pkt_type == PACKET_LOOPBACK)
1296 goto out;
1297
09ad9bc7 1298 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1299 goto out;
1300
40d4e3df
ED		 1301 skb = skb_share_check(skb, GFP_ATOMIC);
1302 if (skb == NULL)
1da177e4
LT		 1303 goto oom;
1304
1305 /* drop any routing info */
adf30907 1306 skb_dst_drop(skb);
1da177e4 1307
84531c24
PO		 1308 /* drop conntrack reference */
1309 nf_reset(skb);
1310
ffbc6111 1311 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1312
98e399f8 1313 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1314
1315 /*
1316 * The SOCK_PACKET socket receives _all_ frames.
1317 */
1318
1319 spkt->spkt_family = dev->type;
1320 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1321 spkt->spkt_protocol = skb->protocol;
1322
1323 /*
1324 * Charge the memory to the socket. This is done specifically
1325 * to prevent sockets using all the memory up.
1326 */
1327
40d4e3df 1328 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1329 return 0;
1330
1331out:
1332 kfree_skb(skb);
1333oom:
1334 return 0;
1335}
1336
1337
1338/*
1339 * Output a raw packet to a device layer. This bypasses all the other
1340 * protocol layers and you must therefore supply it with a complete frame
1341 */
1ce4f28b 1342
1da177e4
LT		 1343static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1344 struct msghdr *msg, size_t len)
1345{
1346 struct sock *sk = sock->sk;
40d4e3df 1347 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1348 struct sk_buff *skb = NULL;
1da177e4 1349 struct net_device *dev;
40d4e3df 1350 __be16 proto = 0;
1da177e4 1351 int err;
3bdc0eba 1352 int extra_len = 0;
1ce4f28b 1353
1da177e4 1354 /*
1ce4f28b 1355 * Get and verify the address.
1da177e4
LT		 1356 */
1357
40d4e3df 1358 if (saddr) {
1da177e4 1359 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1360 return -EINVAL;
1361 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1362 proto = saddr->spkt_protocol;
1363 } else
1364 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1365
1366 /*
1ce4f28b 1367 * Find the device first to size check it
1da177e4
LT		 1368 */
1369
de74e92a 1370 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1371retry:
654d1f8a
ED		 1372 rcu_read_lock();
1373 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1374 err = -ENODEV;
1375 if (dev == NULL)
1376 goto out_unlock;
1ce4f28b 1377
d5e76b0a
DM		 1378 err = -ENETDOWN;
1379 if (!(dev->flags & IFF_UP))
1380 goto out_unlock;
1381
1da177e4 1382 /*
40d4e3df
ED		 1383 * You may not queue a frame bigger than the mtu. This is the lowest level
1384 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1385 */
1ce4f28b 1386
3bdc0eba
BG		 1387 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1388 if (!netif_supports_nofcs(dev)) {
1389 err = -EPROTONOSUPPORT;
1390 goto out_unlock;
1391 }
1392 extra_len = 4; /* We're doing our own CRC */
1393 }
1394
1da177e4 1395 err = -EMSGSIZE;
3bdc0eba 1396 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1397 goto out_unlock;
1398
1a35ca80
ED		 1399 if (!skb) {
1400 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1401 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1402 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1403
1404 rcu_read_unlock();
4ce40912 1405 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1406 if (skb == NULL)
1407 return -ENOBUFS;
1408 /* FIXME: Save some space for broken drivers that write a hard
1409 * header at transmission time by themselves. PPP is the notable
1410 * one here. This should really be fixed at the driver level.
1411 */
1412 skb_reserve(skb, reserved);
1413 skb_reset_network_header(skb);
1414
1415 /* Try to align data part correctly */
1416 if (hhlen) {
1417 skb->data -= hhlen;
1418 skb->tail -= hhlen;
1419 if (len < hhlen)
1420 skb_reset_network_header(skb);
1421 }
1422 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1423 if (err)
1424 goto out_free;
1425 goto retry;
1da177e4
LT		 1426 }
1427
3bdc0eba 1428 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1429 /* Earlier code assumed this would be a VLAN pkt,
1430 * double-check this now that we have the actual
1431 * packet in hand.
1432 */
1433 struct ethhdr *ehdr;
1434 skb_reset_mac_header(skb);
1435 ehdr = eth_hdr(skb);
1436 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1437 err = -EMSGSIZE;
1438 goto out_unlock;
1439 }
1440 }
1a35ca80 1441
1da177e4
LT		 1442 skb->protocol = proto;
1443 skb->dev = dev;
1444 skb->priority = sk->sk_priority;
2d37a186 1445 skb->mark = sk->sk_mark;
2244d07b 1446 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 1447 if (err < 0)
1448 goto out_unlock;
1da177e4 1449
3bdc0eba
BG		 1450 if (unlikely(extra_len == 4))
1451 skb->no_fcs = 1;
1452
1da177e4 1453 dev_queue_xmit(skb);
654d1f8a 1454 rcu_read_unlock();
40d4e3df 1455 return len;
1da177e4 1456
1da177e4 1457out_unlock:
654d1f8a 1458 rcu_read_unlock();
1a35ca80
ED		 1459out_free:
1460 kfree_skb(skb);
1da177e4
LT		 1461 return err;
1462}
1da177e4 1463
eea49cc9 1464static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1465 const struct sock *sk,
dbcb5855 1466 unsigned int res)
1da177e4
LT		 1467{
1468 struct sk_filter *filter;
fda9ef5d 1469
80f8f102
ED		 1470 rcu_read_lock();
1471 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1472 if (filter != NULL)
0a14842f 1473 res = SK_RUN_FILTER(filter, skb);
80f8f102 1474 rcu_read_unlock();
1da177e4 1475
dbcb5855 1476 return res;
1da177e4
LT		 1477}
1478
1479/*
62ab0812
ED		 1480 * This function makes lazy skb cloning in hope that most of packets
1481 * are discarded by BPF.
1482 *
1483 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1484 * and skb->cb are mangled. It works because (and until) packets
1485 * falling here are owned by current CPU. Output packets are cloned
1486 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1487 * sequencially, so that if we return skb to original state on exit,
1488 * we will not harm anyone.
1da177e4
LT		 1489 */
1490
40d4e3df
ED		 1491static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1492 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1493{
1494 struct sock *sk;
1495 struct sockaddr_ll *sll;
1496 struct packet_sock *po;
40d4e3df 1497 u8 *skb_head = skb->data;
1da177e4 1498 int skb_len = skb->len;
dbcb5855 1499 unsigned int snaplen, res;
1da177e4
LT		 1500
1501 if (skb->pkt_type == PACKET_LOOPBACK)
1502 goto drop;
1503
1504 sk = pt->af_packet_priv;
1505 po = pkt_sk(sk);
1506
09ad9bc7 1507 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1508 goto drop;
1509
1da177e4
LT		 1510 skb->dev = dev;
1511
3b04ddde 1512 if (dev->header_ops) {
1da177e4 1513 /* The device has an explicit notion of ll header,
62ab0812
ED		 1514 * exported to higher levels.
1515 *
1516 * Otherwise, the device hides details of its frame
1517 * structure, so that corresponding packet head is
1518 * never delivered to user.
1da177e4
LT		 1519 */
1520 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1521 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1522 else if (skb->pkt_type == PACKET_OUTGOING) {
1523 /* Special case: outgoing packets have ll header at head */
bbe735e4 1524 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1525 }
1526 }
1527
1528 snaplen = skb->len;
1529
dbcb5855
DM		 1530 res = run_filter(skb, sk, snaplen);
1531 if (!res)
fda9ef5d 1532 goto drop_n_restore;
dbcb5855
DM		 1533 if (snaplen > res)
1534 snaplen = res;
1da177e4 1535
0fd7bac6 1536 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1537 goto drop_n_acct;
1538
1539 if (skb_shared(skb)) {
1540 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1541 if (nskb == NULL)
1542 goto drop_n_acct;
1543
1544 if (skb_head != skb->data) {
1545 skb->data = skb_head;
1546 skb->len = skb_len;
1547 }
abc4e4fa 1548 consume_skb(skb);
1da177e4
LT		 1549 skb = nskb;
1550 }
1551
ffbc6111
HX		 1552 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1553 sizeof(skb->cb));
1554
1555 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1556 sll->sll_family = AF_PACKET;
1557 sll->sll_hatype = dev->type;
1558 sll->sll_protocol = skb->protocol;
1559 sll->sll_pkttype = skb->pkt_type;
8032b464 1560 if (unlikely(po->origdev))
80feaacb
PWJ		 1561 sll->sll_ifindex = orig_dev->ifindex;
1562 else
1563 sll->sll_ifindex = dev->ifindex;
1da177e4 1564
b95cce35 1565 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1566
ffbc6111 1567 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1568
1da177e4
LT		 1569 if (pskb_trim(skb, snaplen))
1570 goto drop_n_acct;
1571
1572 skb_set_owner_r(skb, sk);
1573 skb->dev = NULL;
adf30907 1574 skb_dst_drop(skb);
1da177e4 1575
84531c24
PO		 1576 /* drop conntrack reference */
1577 nf_reset(skb);
1578
1da177e4
LT		 1579 spin_lock(&sk->sk_receive_queue.lock);
1580 po->stats.tp_packets++;
3b885787 1581 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1582 __skb_queue_tail(&sk->sk_receive_queue, skb);
1583 spin_unlock(&sk->sk_receive_queue.lock);
1584 sk->sk_data_ready(sk, skb->len);
1585 return 0;
1586
1587drop_n_acct:
7091fbd8
WB		 1588 spin_lock(&sk->sk_receive_queue.lock);
1589 po->stats.tp_drops++;
1590 atomic_inc(&sk->sk_drops);
1591 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1592
1593drop_n_restore:
1594 if (skb_head != skb->data && skb_shared(skb)) {
1595 skb->data = skb_head;
1596 skb->len = skb_len;
1597 }
1598drop:
ead2ceb0 1599 consume_skb(skb);
1da177e4
LT		 1600 return 0;
1601}
1602
40d4e3df
ED		 1603static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1604 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1605{
1606 struct sock *sk;
1607 struct packet_sock *po;
1608 struct sockaddr_ll *sll;
bbd6ef87
PM		 1609 union {
1610 struct tpacket_hdr *h1;
1611 struct tpacket2_hdr *h2;
f6fb8f10 1612 struct tpacket3_hdr *h3;
bbd6ef87
PM		 1613 void *raw;
1614 } h;
40d4e3df 1615 u8 *skb_head = skb->data;
1da177e4 1616 int skb_len = skb->len;
dbcb5855 1617 unsigned int snaplen, res;
f6fb8f10 1618 unsigned long status = TP_STATUS_USER;
bbd6ef87 1619 unsigned short macoff, netoff, hdrlen;
1da177e4 1620 struct sk_buff *copy_skb = NULL;
b7aa0bf7 1621 struct timeval tv;
bbd6ef87 1622 struct timespec ts;
614f60fa 1623 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1da177e4
LT		 1624
1625 if (skb->pkt_type == PACKET_LOOPBACK)
1626 goto drop;
1627
1628 sk = pt->af_packet_priv;
1629 po = pkt_sk(sk);
1630
09ad9bc7 1631 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1632 goto drop;
1633
3b04ddde 1634 if (dev->header_ops) {
1da177e4 1635 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1636 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1637 else if (skb->pkt_type == PACKET_OUTGOING) {
1638 /* Special case: outgoing packets have ll header at head */
bbe735e4 1639 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1640 }
1641 }
1642
8dc41944
HX		 1643 if (skb->ip_summed == CHECKSUM_PARTIAL)
1644 status |= TP_STATUS_CSUMNOTREADY;
1645
1da177e4
LT		 1646 snaplen = skb->len;
1647
dbcb5855
DM		 1648 res = run_filter(skb, sk, snaplen);
1649 if (!res)
fda9ef5d 1650 goto drop_n_restore;
dbcb5855
DM		 1651 if (snaplen > res)
1652 snaplen = res;
1da177e4
LT		 1653
1654 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1655 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1656 po->tp_reserve;
1da177e4 1657 } else {
95c96174 1658 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1659 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1660 (maclen < 16 ? 16 : maclen)) +
1661 po->tp_reserve;
1da177e4
LT		 1662 macoff = netoff - maclen;
1663 }
f6fb8f10 1664 if (po->tp_version <= TPACKET_V2) {
1665 if (macoff + snaplen > po->rx_ring.frame_size) {
1666 if (po->copy_thresh &&
0fd7bac6 1667 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1668 if (skb_shared(skb)) {
1669 copy_skb = skb_clone(skb, GFP_ATOMIC);
1670 } else {
1671 copy_skb = skb_get(skb);
1672 skb_head = skb->data;
1673 }
1674 if (copy_skb)
1675 skb_set_owner_r(copy_skb, sk);
1da177e4 1676 }
f6fb8f10 1677 snaplen = po->rx_ring.frame_size - macoff;
1678 if ((int)snaplen < 0)
1679 snaplen = 0;
1da177e4 1680 }
1da177e4 1681 }
1da177e4 1682 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1683 h.raw = packet_current_rx_frame(po, skb,
1684 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1685 if (!h.raw)
1da177e4 1686 goto ring_is_full;
f6fb8f10 1687 if (po->tp_version <= TPACKET_V2) {
1688 packet_increment_rx_head(po, &po->rx_ring);
1689 /*
1690 * LOSING will be reported till you read the stats,
1691 * because it's COR - Clear On Read.
1692 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1693 * at packet level.
1694 */
1695 if (po->stats.tp_drops)
1696 status |= TP_STATUS_LOSING;
1697 }
1da177e4
LT		 1698 po->stats.tp_packets++;
1699 if (copy_skb) {
1700 status |= TP_STATUS_COPY;
1701 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1702 }
1da177e4
LT		 1703 spin_unlock(&sk->sk_receive_queue.lock);
1704
bbd6ef87 1705 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1da177e4 1706
bbd6ef87
PM		 1707 switch (po->tp_version) {
1708 case TPACKET_V1:
1709 h.h1->tp_len = skb->len;
1710 h.h1->tp_snaplen = snaplen;
1711 h.h1->tp_mac = macoff;
1712 h.h1->tp_net = netoff;
614f60fa
SM		 1713 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1714 && shhwtstamps->syststamp.tv64)
1715 tv = ktime_to_timeval(shhwtstamps->syststamp);
1716 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1717 && shhwtstamps->hwtstamp.tv64)
1718 tv = ktime_to_timeval(shhwtstamps->hwtstamp);
1719 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1720 tv = ktime_to_timeval(skb->tstamp);
1721 else
1722 do_gettimeofday(&tv);
1723 h.h1->tp_sec = tv.tv_sec;
1724 h.h1->tp_usec = tv.tv_usec;
1725 hdrlen = sizeof(*h.h1);
1726 break;
1727 case TPACKET_V2:
1728 h.h2->tp_len = skb->len;
1729 h.h2->tp_snaplen = snaplen;
1730 h.h2->tp_mac = macoff;
1731 h.h2->tp_net = netoff;
614f60fa
SM		 1732 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1733 && shhwtstamps->syststamp.tv64)
1734 ts = ktime_to_timespec(shhwtstamps->syststamp);
1735 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1736 && shhwtstamps->hwtstamp.tv64)
1737 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1738 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1739 ts = ktime_to_timespec(skb->tstamp);
1740 else
1741 getnstimeofday(&ts);
1742 h.h2->tp_sec = ts.tv_sec;
1743 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1744 if (vlan_tx_tag_present(skb)) {
1745 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1746 status |= TP_STATUS_VLAN_VALID;
1747 } else {
1748 h.h2->tp_vlan_tci = 0;
1749 }
13fcb7bd 1750 h.h2->tp_padding = 0;
bbd6ef87
PM		 1751 hdrlen = sizeof(*h.h2);
1752 break;
f6fb8f10 1753 case TPACKET_V3:
1754 /* tp_nxt_offset,vlan are already populated above.
1755 * So DONT clear those fields here
1756 */
1757 h.h3->tp_status |= status;
1758 h.h3->tp_len = skb->len;
1759 h.h3->tp_snaplen = snaplen;
1760 h.h3->tp_mac = macoff;
1761 h.h3->tp_net = netoff;
1762 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1763 && shhwtstamps->syststamp.tv64)
1764 ts = ktime_to_timespec(shhwtstamps->syststamp);
1765 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1766 && shhwtstamps->hwtstamp.tv64)
1767 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1768 else if (skb->tstamp.tv64)
1769 ts = ktime_to_timespec(skb->tstamp);
1770 else
1771 getnstimeofday(&ts);
1772 h.h3->tp_sec = ts.tv_sec;
1773 h.h3->tp_nsec = ts.tv_nsec;
1774 hdrlen = sizeof(*h.h3);
1775 break;
bbd6ef87
PM		 1776 default:
1777 BUG();
1778 }
1da177e4 1779
bbd6ef87 1780 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1781 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1782 sll->sll_family = AF_PACKET;
1783 sll->sll_hatype = dev->type;
1784 sll->sll_protocol = skb->protocol;
1785 sll->sll_pkttype = skb->pkt_type;
8032b464 1786 if (unlikely(po->origdev))
80feaacb
PWJ		 1787 sll->sll_ifindex = orig_dev->ifindex;
1788 else
1789 sll->sll_ifindex = dev->ifindex;
1da177e4 1790
e16aa207 1791 smp_mb();
f6dafa95 1792#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1793 {
0af55bb5
CG		 1794 u8 *start, *end;
1795
f6fb8f10 1796 if (po->tp_version <= TPACKET_V2) {
1797 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1798 + macoff + snaplen);
1799 for (start = h.raw; start < end; start += PAGE_SIZE)
1800 flush_dcache_page(pgv_to_page(start));
1801 }
cc9f01b2 1802 smp_wmb();
1da177e4 1803 }
f6dafa95 1804#endif
f6fb8f10 1805 if (po->tp_version <= TPACKET_V2)
1806 __packet_set_status(po, h.raw, status);
1807 else
1808 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1809
1810 sk->sk_data_ready(sk, 0);
1811
1812drop_n_restore:
1813 if (skb_head != skb->data && skb_shared(skb)) {
1814 skb->data = skb_head;
1815 skb->len = skb_len;
1816 }
1817drop:
1ce4f28b 1818 kfree_skb(skb);
1da177e4
LT		 1819 return 0;
1820
1821ring_is_full:
1822 po->stats.tp_drops++;
1823 spin_unlock(&sk->sk_receive_queue.lock);
1824
1825 sk->sk_data_ready(sk, 0);
acb5d75b 1826 kfree_skb(copy_skb);
1da177e4
LT		 1827 goto drop_n_restore;
1828}
1829
69e3c75f
JB		 1830static void tpacket_destruct_skb(struct sk_buff *skb)
1831{
1832 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1833 void *ph;
1da177e4 1834
69e3c75f
JB		 1835 if (likely(po->tx_ring.pg_vec)) {
1836 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1837 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1838 atomic_dec(&po->tx_ring.pending);
1839 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
1840 }
1841
1842 sock_wfree(skb);
1843}
1844
40d4e3df
ED		 1845static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1846 void *frame, struct net_device *dev, int size_max,
ae641949 1847 __be16 proto, unsigned char *addr, int hlen)
69e3c75f
JB		 1848{
1849 union {
1850 struct tpacket_hdr *h1;
1851 struct tpacket2_hdr *h2;
1852 void *raw;
1853 } ph;
1854 int to_write, offset, len, tp_len, nr_frags, len_max;
1855 struct socket *sock = po->sk.sk_socket;
1856 struct page *page;
1857 void *data;
1858 int err;
1859
1860 ph.raw = frame;
1861
1862 skb->protocol = proto;
1863 skb->dev = dev;
1864 skb->priority = po->sk.sk_priority;
2d37a186 1865 skb->mark = po->sk.sk_mark;
69e3c75f
JB		 1866 skb_shinfo(skb)->destructor_arg = ph.raw;
1867
1868 switch (po->tp_version) {
1869 case TPACKET_V2:
1870 tp_len = ph.h2->tp_len;
1871 break;
1872 default:
1873 tp_len = ph.h1->tp_len;
1874 break;
1875 }
1876 if (unlikely(tp_len > size_max)) {
40d4e3df 1877 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1878 return -EMSGSIZE;
1879 }
1880
ae641949 1881 skb_reserve(skb, hlen);
69e3c75f
JB		 1882 skb_reset_network_header(skb);
1883
1884 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1885 to_write = tp_len;
1886
1887 if (sock->type == SOCK_DGRAM) {
1888 err = dev_hard_header(skb, dev, ntohs(proto), addr,
1889 NULL, tp_len);
1890 if (unlikely(err < 0))
1891 return -EINVAL;
40d4e3df 1892 } else if (dev->hard_header_len) {
69e3c75f
JB		 1893 /* net device doesn't like empty head */
1894 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 1895 pr_err("packet size is too short (%d < %d)\n",
1896 tp_len, dev->hard_header_len);
69e3c75f
JB		 1897 return -EINVAL;
1898 }
1899
1900 skb_push(skb, dev->hard_header_len);
1901 err = skb_store_bits(skb, 0, data,
1902 dev->hard_header_len);
1903 if (unlikely(err))
1904 return err;
1905
1906 data += dev->hard_header_len;
1907 to_write -= dev->hard_header_len;
1908 }
1909
69e3c75f
JB		 1910 offset = offset_in_page(data);
1911 len_max = PAGE_SIZE - offset;
1912 len = ((to_write > len_max) ? len_max : to_write);
1913
1914 skb->data_len = to_write;
1915 skb->len += to_write;
1916 skb->truesize += to_write;
1917 atomic_add(to_write, &po->sk.sk_wmem_alloc);
1918
1919 while (likely(to_write)) {
1920 nr_frags = skb_shinfo(skb)->nr_frags;
1921
1922 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 1923 pr_err("Packet exceed the number of skb frags(%lu)\n",
1924 MAX_SKB_FRAGS);
69e3c75f
JB		 1925 return -EFAULT;
1926 }
1927
0af55bb5
CG		 1928 page = pgv_to_page(data);
1929 data += len;
69e3c75f
JB		 1930 flush_dcache_page(page);
1931 get_page(page);
0af55bb5 1932 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 1933 to_write -= len;
1934 offset = 0;
1935 len_max = PAGE_SIZE;
1936 len = ((to_write > len_max) ? len_max : to_write);
1937 }
1938
1939 return tp_len;
1940}
1941
1942static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
1943{
69e3c75f
JB		 1944 struct sk_buff *skb;
1945 struct net_device *dev;
1946 __be16 proto;
827d9780
BG		 1947 bool need_rls_dev = false;
1948 int err, reserve = 0;
40d4e3df
ED		 1949 void *ph;
1950 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 1951 int tp_len, size_max;
1952 unsigned char *addr;
1953 int len_sum = 0;
9e67030a 1954 int status = TP_STATUS_AVAILABLE;
ae641949 1955 int hlen, tlen;
69e3c75f 1956
69e3c75f
JB		 1957 mutex_lock(&po->pg_vec_lock);
1958
69e3c75f 1959 if (saddr == NULL) {
827d9780 1960 dev = po->prot_hook.dev;
69e3c75f
JB		 1961 proto = po->num;
1962 addr = NULL;
1963 } else {
1964 err = -EINVAL;
1965 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
1966 goto out;
1967 if (msg->msg_namelen < (saddr->sll_halen
1968 + offsetof(struct sockaddr_ll,
1969 sll_addr)))
1970 goto out;
69e3c75f
JB		 1971 proto = saddr->sll_protocol;
1972 addr = saddr->sll_addr;
827d9780
BG		 1973 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
1974 need_rls_dev = true;
69e3c75f
JB		 1975 }
1976
69e3c75f
JB		 1977 err = -ENXIO;
1978 if (unlikely(dev == NULL))
1979 goto out;
1980
1981 reserve = dev->hard_header_len;
1982
1983 err = -ENETDOWN;
1984 if (unlikely(!(dev->flags & IFF_UP)))
1985 goto out_put;
1986
1987 size_max = po->tx_ring.frame_size
b5dd884e 1988 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 1989
1990 if (size_max > dev->mtu + reserve)
1991 size_max = dev->mtu + reserve;
1992
1993 do {
1994 ph = packet_current_frame(po, &po->tx_ring,
1995 TP_STATUS_SEND_REQUEST);
1996
1997 if (unlikely(ph == NULL)) {
1998 schedule();
1999 continue;
2000 }
2001
2002 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2003 hlen = LL_RESERVED_SPACE(dev);
2004 tlen = dev->needed_tailroom;
69e3c75f 2005 skb = sock_alloc_send_skb(&po->sk,
ae641949 2006 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2007 0, &err);
2008
2009 if (unlikely(skb == NULL))
2010 goto out_status;
2011
2012 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2013 addr, hlen);
69e3c75f
JB		 2014
2015 if (unlikely(tp_len < 0)) {
2016 if (po->tp_loss) {
2017 __packet_set_status(po, ph,
2018 TP_STATUS_AVAILABLE);
2019 packet_increment_head(&po->tx_ring);
2020 kfree_skb(skb);
2021 continue;
2022 } else {
2023 status = TP_STATUS_WRONG_FORMAT;
2024 err = tp_len;
2025 goto out_status;
2026 }
2027 }
2028
2029 skb->destructor = tpacket_destruct_skb;
2030 __packet_set_status(po, ph, TP_STATUS_SENDING);
2031 atomic_inc(&po->tx_ring.pending);
2032
2033 status = TP_STATUS_SEND_REQUEST;
2034 err = dev_queue_xmit(skb);
eb70df13
JP		 2035 if (unlikely(err > 0)) {
2036 err = net_xmit_errno(err);
2037 if (err && __packet_get_status(po, ph) ==
2038 TP_STATUS_AVAILABLE) {
2039 /* skb was destructed already */
2040 skb = NULL;
2041 goto out_status;
2042 }
2043 /*
2044 * skb was dropped but not destructed yet;
2045 * let's treat it like congestion or err < 0
2046 */
2047 err = 0;
2048 }
69e3c75f
JB		 2049 packet_increment_head(&po->tx_ring);
2050 len_sum += tp_len;
f64f9e71
JP		 2051 } while (likely((ph != NULL) ||
2052 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2053 (atomic_read(&po->tx_ring.pending))))
2054 );
69e3c75f
JB		 2055
2056 err = len_sum;
2057 goto out_put;
2058
69e3c75f
JB		 2059out_status:
2060 __packet_set_status(po, ph, status);
2061 kfree_skb(skb);
2062out_put:
827d9780
BG		 2063 if (need_rls_dev)
2064 dev_put(dev);
69e3c75f
JB		 2065out:
2066 mutex_unlock(&po->pg_vec_lock);
2067 return err;
2068}
69e3c75f 2069
eea49cc9
OJ		 2070static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2071 size_t reserve, size_t len,
2072 size_t linear, int noblock,
2073 int *err)
bfd5f4a3
SS		 2074{
2075 struct sk_buff *skb;
2076
2077 /* Under a page? Don't bother with paged skb. */
2078 if (prepad + len < PAGE_SIZE || !linear)
2079 linear = len;
2080
2081 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2082 err);
2083 if (!skb)
2084 return NULL;
2085
2086 skb_reserve(skb, reserve);
2087 skb_put(skb, linear);
2088 skb->data_len = len - linear;
2089 skb->len += len - linear;
2090
2091 return skb;
2092}
2093
69e3c75f 2094static int packet_snd(struct socket *sock,
1da177e4
LT		 2095 struct msghdr *msg, size_t len)
2096{
2097 struct sock *sk = sock->sk;
40d4e3df 2098 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2099 struct sk_buff *skb;
2100 struct net_device *dev;
0e11c91e 2101 __be16 proto;
827d9780 2102 bool need_rls_dev = false;
1da177e4 2103 unsigned char *addr;
827d9780 2104 int err, reserve = 0;
bfd5f4a3
SS		 2105 struct virtio_net_hdr vnet_hdr = { 0 };
2106 int offset = 0;
2107 int vnet_hdr_len;
2108 struct packet_sock *po = pkt_sk(sk);
2109 unsigned short gso_type = 0;
ae641949 2110 int hlen, tlen;
3bdc0eba 2111 int extra_len = 0;
1da177e4
LT		 2112
2113 /*
1ce4f28b 2114 * Get and verify the address.
1da177e4 2115 */
1ce4f28b 2116
1da177e4 2117 if (saddr == NULL) {
827d9780 2118 dev = po->prot_hook.dev;
1da177e4
LT		 2119 proto = po->num;
2120 addr = NULL;
2121 } else {
2122 err = -EINVAL;
2123 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2124 goto out;
0fb375fb
EB		 2125 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2126 goto out;
1da177e4
LT		 2127 proto = saddr->sll_protocol;
2128 addr = saddr->sll_addr;
827d9780
BG		 2129 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2130 need_rls_dev = true;
1da177e4
LT		 2131 }
2132
1da177e4
LT		 2133 err = -ENXIO;
2134 if (dev == NULL)
2135 goto out_unlock;
2136 if (sock->type == SOCK_RAW)
2137 reserve = dev->hard_header_len;
2138
d5e76b0a
DM		 2139 err = -ENETDOWN;
2140 if (!(dev->flags & IFF_UP))
2141 goto out_unlock;
2142
bfd5f4a3
SS		 2143 if (po->has_vnet_hdr) {
2144 vnet_hdr_len = sizeof(vnet_hdr);
2145
2146 err = -EINVAL;
2147 if (len < vnet_hdr_len)
2148 goto out_unlock;
2149
2150 len -= vnet_hdr_len;
2151
2152 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2153 vnet_hdr_len);
2154 if (err < 0)
2155 goto out_unlock;
2156
2157 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2158 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2159 vnet_hdr.hdr_len))
2160 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2161 vnet_hdr.csum_offset + 2;
2162
2163 err = -EINVAL;
2164 if (vnet_hdr.hdr_len > len)
2165 goto out_unlock;
2166
2167 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2168 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2169 case VIRTIO_NET_HDR_GSO_TCPV4:
2170 gso_type = SKB_GSO_TCPV4;
2171 break;
2172 case VIRTIO_NET_HDR_GSO_TCPV6:
2173 gso_type = SKB_GSO_TCPV6;
2174 break;
2175 case VIRTIO_NET_HDR_GSO_UDP:
2176 gso_type = SKB_GSO_UDP;
2177 break;
2178 default:
2179 goto out_unlock;
2180 }
2181
2182 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2183 gso_type |= SKB_GSO_TCP_ECN;
2184
2185 if (vnet_hdr.gso_size == 0)
2186 goto out_unlock;
2187
2188 }
2189 }
2190
3bdc0eba
BG		 2191 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2192 if (!netif_supports_nofcs(dev)) {
2193 err = -EPROTONOSUPPORT;
2194 goto out_unlock;
2195 }
2196 extra_len = 4; /* We're doing our own CRC */
2197 }
2198
1da177e4 2199 err = -EMSGSIZE;
3bdc0eba 2200 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2201 goto out_unlock;
2202
bfd5f4a3 2203 err = -ENOBUFS;
ae641949
HX		 2204 hlen = LL_RESERVED_SPACE(dev);
2205 tlen = dev->needed_tailroom;
2206 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2207 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2208 if (skb == NULL)
1da177e4
LT		 2209 goto out_unlock;
2210
bfd5f4a3 2211 skb_set_network_header(skb, reserve);
1da177e4 2212
0c4e8581
SH		 2213 err = -EINVAL;
2214 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2215 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2216 goto out_free;
1da177e4
LT		 2217
2218 /* Returns -EFAULT on error */
bfd5f4a3 2219 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2220 if (err)
2221 goto out_free;
2244d07b 2222 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 2223 if (err < 0)
2224 goto out_free;
1da177e4 2225
3bdc0eba 2226 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
57f89bfa
BG		 2227 /* Earlier code assumed this would be a VLAN pkt,
2228 * double-check this now that we have the actual
2229 * packet in hand.
2230 */
2231 struct ethhdr *ehdr;
2232 skb_reset_mac_header(skb);
2233 ehdr = eth_hdr(skb);
2234 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2235 err = -EMSGSIZE;
2236 goto out_free;
2237 }
2238 }
2239
1da177e4
LT		 2240 skb->protocol = proto;
2241 skb->dev = dev;
2242 skb->priority = sk->sk_priority;
2d37a186 2243 skb->mark = sk->sk_mark;
1da177e4 2244
bfd5f4a3
SS		 2245 if (po->has_vnet_hdr) {
2246 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2247 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2248 vnet_hdr.csum_offset)) {
2249 err = -EINVAL;
2250 goto out_free;
2251 }
2252 }
2253
2254 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2255 skb_shinfo(skb)->gso_type = gso_type;
2256
2257 /* Header must be checked, and gso_segs computed. */
2258 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2259 skb_shinfo(skb)->gso_segs = 0;
2260
2261 len += vnet_hdr_len;
2262 }
2263
3bdc0eba
BG		 2264 if (unlikely(extra_len == 4))
2265 skb->no_fcs = 1;
2266
1da177e4
LT		 2267 /*
2268 * Now send it
2269 */
2270
2271 err = dev_queue_xmit(skb);
2272 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2273 goto out_unlock;
2274
827d9780
BG		 2275 if (need_rls_dev)
2276 dev_put(dev);
1da177e4 2277
40d4e3df 2278 return len;
1da177e4
LT		 2279
2280out_free:
2281 kfree_skb(skb);
2282out_unlock:
827d9780 2283 if (dev && need_rls_dev)
1da177e4
LT		 2284 dev_put(dev);
2285out:
2286 return err;
2287}
2288
69e3c75f
JB		 2289static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2290 struct msghdr *msg, size_t len)
2291{
69e3c75f
JB		 2292 struct sock *sk = sock->sk;
2293 struct packet_sock *po = pkt_sk(sk);
2294 if (po->tx_ring.pg_vec)
2295 return tpacket_snd(po, msg);
2296 else
69e3c75f
JB		 2297 return packet_snd(sock, msg, len);
2298}
2299
1da177e4
LT		 2300/*
2301 * Close a PACKET socket. This is fairly simple. We immediately go
2302 * to 'closed' state and remove our protocol entry in the device list.
2303 */
2304
2305static int packet_release(struct socket *sock)
2306{
2307 struct sock *sk = sock->sk;
2308 struct packet_sock *po;
d12d01d6 2309 struct net *net;
f6fb8f10 2310 union tpacket_req_u req_u;
1da177e4
LT		 2311
2312 if (!sk)
2313 return 0;
2314
3b1e0a65 2315 net = sock_net(sk);
1da177e4
LT		 2316 po = pkt_sk(sk);
2317
0fa7fa98 2318 mutex_lock(&net->packet.sklist_lock);
808f5114 2319 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2320 mutex_unlock(&net->packet.sklist_lock);
2321
2322 preempt_disable();
920de804 2323 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2324 preempt_enable();
1da177e4 2325
808f5114 2326 spin_lock(&po->bind_lock);
ce06b03e 2327 unregister_prot_hook(sk, false);
160ff18a
BG		 2328 if (po->prot_hook.dev) {
2329 dev_put(po->prot_hook.dev);
2330 po->prot_hook.dev = NULL;
2331 }
808f5114 2332 spin_unlock(&po->bind_lock);
1da177e4 2333
1da177e4 2334 packet_flush_mclist(sk);
1da177e4 2335
f6fb8f10 2336 memset(&req_u, 0, sizeof(req_u));
69e3c75f
JB		 2337
2338 if (po->rx_ring.pg_vec)
f6fb8f10 2339 packet_set_ring(sk, &req_u, 1, 0);
69e3c75f
JB		 2340
2341 if (po->tx_ring.pg_vec)
f6fb8f10 2342 packet_set_ring(sk, &req_u, 1, 1);
1da177e4 2343
dc99f600
DM		 2344 fanout_release(sk);
2345
808f5114 2346 synchronize_net();
1da177e4
LT		 2347 /*
2348 * Now the socket is dead. No more input will appear.
2349 */
1da177e4
LT		 2350 sock_orphan(sk);
2351 sock->sk = NULL;
2352
2353 /* Purge queues */
2354
2355 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2356 sk_refcnt_debug_release(sk);
1da177e4
LT		 2357
2358 sock_put(sk);
2359 return 0;
2360}
2361
2362/*
2363 * Attach a packet hook.
2364 */
2365
0e11c91e 2366static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2367{
2368 struct packet_sock *po = pkt_sk(sk);
dc99f600 2369
aef950b4
WY		 2370 if (po->fanout) {
2371 if (dev)
2372 dev_put(dev);
2373
dc99f600 2374 return -EINVAL;
aef950b4 2375 }
1da177e4
LT		 2376
2377 lock_sock(sk);
2378
2379 spin_lock(&po->bind_lock);
ce06b03e 2380 unregister_prot_hook(sk, true);
1da177e4
LT		 2381 po->num = protocol;
2382 po->prot_hook.type = protocol;
160ff18a
BG		 2383 if (po->prot_hook.dev)
2384 dev_put(po->prot_hook.dev);
1da177e4
LT		 2385 po->prot_hook.dev = dev;
2386
2387 po->ifindex = dev ? dev->ifindex : 0;
2388
2389 if (protocol == 0)
2390 goto out_unlock;
2391
be85d4ad 2392 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2393 register_prot_hook(sk);
be85d4ad
UT		 2394 } else {
2395 sk->sk_err = ENETDOWN;
2396 if (!sock_flag(sk, SOCK_DEAD))
2397 sk->sk_error_report(sk);
1da177e4
LT		 2398 }
2399
2400out_unlock:
2401 spin_unlock(&po->bind_lock);
2402 release_sock(sk);
2403 return 0;
2404}
2405
2406/*
2407 * Bind a packet socket to a device
2408 */
2409
40d4e3df
ED		 2410static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2411 int addr_len)
1da177e4 2412{
40d4e3df 2413 struct sock *sk = sock->sk;
1da177e4
LT		 2414 char name[15];
2415 struct net_device *dev;
2416 int err = -ENODEV;
1ce4f28b 2417
1da177e4
LT		 2418 /*
2419 * Check legality
2420 */
1ce4f28b 2421
8ae55f04 2422 if (addr_len != sizeof(struct sockaddr))
1da177e4 2423 return -EINVAL;
40d4e3df 2424 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2425
3b1e0a65 2426 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2427 if (dev)
1da177e4 2428 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2429 return err;
2430}
1da177e4
LT		 2431
2432static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2433{
40d4e3df
ED		 2434 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2435 struct sock *sk = sock->sk;
1da177e4
LT		 2436 struct net_device *dev = NULL;
2437 int err;
2438
2439
2440 /*
2441 * Check legality
2442 */
1ce4f28b 2443
1da177e4
LT		 2444 if (addr_len < sizeof(struct sockaddr_ll))
2445 return -EINVAL;
2446 if (sll->sll_family != AF_PACKET)
2447 return -EINVAL;
2448
2449 if (sll->sll_ifindex) {
2450 err = -ENODEV;
3b1e0a65 2451 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2452 if (dev == NULL)
2453 goto out;
2454 }
2455 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2456
2457out:
2458 return err;
2459}
2460
2461static struct proto packet_proto = {
2462 .name = "PACKET",
2463 .owner = THIS_MODULE,
2464 .obj_size = sizeof(struct packet_sock),
2465};
2466
2467/*
1ce4f28b 2468 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2469 */
2470
3f378b68
EP		 2471static int packet_create(struct net *net, struct socket *sock, int protocol,
2472 int kern)
1da177e4
LT		 2473{
2474 struct sock *sk;
2475 struct packet_sock *po;
0e11c91e 2476 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2477 int err;
2478
2479 if (!capable(CAP_NET_RAW))
2480 return -EPERM;
be02097c
DM		 2481 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2482 sock->type != SOCK_PACKET)
1da177e4
LT		 2483 return -ESOCKTNOSUPPORT;
2484
2485 sock->state = SS_UNCONNECTED;
2486
2487 err = -ENOBUFS;
6257ff21 2488 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2489 if (sk == NULL)
2490 goto out;
2491
2492 sock->ops = &packet_ops;
1da177e4
LT		 2493 if (sock->type == SOCK_PACKET)
2494 sock->ops = &packet_ops_spkt;
be02097c 2495
1da177e4
LT		 2496 sock_init_data(sock, sk);
2497
2498 po = pkt_sk(sk);
2499 sk->sk_family = PF_PACKET;
0e11c91e 2500 po->num = proto;
1da177e4
LT		 2501
2502 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2503 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2504
2505 /*
2506 * Attach a protocol block
2507 */
2508
2509 spin_lock_init(&po->bind_lock);
905db440 2510 mutex_init(&po->pg_vec_lock);
1da177e4 2511 po->prot_hook.func = packet_rcv;
be02097c 2512
1da177e4
LT		 2513 if (sock->type == SOCK_PACKET)
2514 po->prot_hook.func = packet_rcv_spkt;
be02097c 2515
1da177e4
LT		 2516 po->prot_hook.af_packet_priv = sk;
2517
0e11c91e
AV		 2518 if (proto) {
2519 po->prot_hook.type = proto;
ce06b03e 2520 register_prot_hook(sk);
1da177e4
LT		 2521 }
2522
0fa7fa98 2523 mutex_lock(&net->packet.sklist_lock);
808f5114 2524 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2525 mutex_unlock(&net->packet.sklist_lock);
2526
2527 preempt_disable();
3680453c 2528 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2529 preempt_enable();
808f5114 2530
40d4e3df 2531 return 0;
1da177e4
LT		 2532out:
2533 return err;
2534}
2535
ed85b565
RC		 2536static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2537{
2538 struct sock_exterr_skb *serr;
2539 struct sk_buff *skb, *skb2;
2540 int copied, err;
2541
2542 err = -EAGAIN;
2543 skb = skb_dequeue(&sk->sk_error_queue);
2544 if (skb == NULL)
2545 goto out;
2546
2547 copied = skb->len;
2548 if (copied > len) {
2549 msg->msg_flags |= MSG_TRUNC;
2550 copied = len;
2551 }
2552 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2553 if (err)
2554 goto out_free_skb;
2555
2556 sock_recv_timestamp(msg, sk, skb);
2557
2558 serr = SKB_EXT_ERR(skb);
2559 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2560 sizeof(serr->ee), &serr->ee);
2561
2562 msg->msg_flags |= MSG_ERRQUEUE;
2563 err = copied;
2564
2565 /* Reset and regenerate socket error */
2566 spin_lock_bh(&sk->sk_error_queue.lock);
2567 sk->sk_err = 0;
2568 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2569 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2570 spin_unlock_bh(&sk->sk_error_queue.lock);
2571 sk->sk_error_report(sk);
2572 } else
2573 spin_unlock_bh(&sk->sk_error_queue.lock);
2574
2575out_free_skb:
2576 kfree_skb(skb);
2577out:
2578 return err;
2579}
2580
1da177e4
LT		 2581/*
2582 * Pull a packet from our receive queue and hand it to the user.
2583 * If necessary we block.
2584 */
2585
2586static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2587 struct msghdr *msg, size_t len, int flags)
2588{
2589 struct sock *sk = sock->sk;
2590 struct sk_buff *skb;
2591 int copied, err;
0fb375fb 2592 struct sockaddr_ll *sll;
bfd5f4a3 2593 int vnet_hdr_len = 0;
1da177e4
LT		 2594
2595 err = -EINVAL;
ed85b565 2596 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2597 goto out;
2598
2599#if 0
2600 /* What error should we return now? EUNATTACH? */
2601 if (pkt_sk(sk)->ifindex < 0)
2602 return -ENODEV;
2603#endif
2604
ed85b565
RC		 2605 if (flags & MSG_ERRQUEUE) {
2606 err = packet_recv_error(sk, msg, len);
2607 goto out;
2608 }
2609
1da177e4
LT		 2610 /*
2611 * Call the generic datagram receiver. This handles all sorts
2612 * of horrible races and re-entrancy so we can forget about it
2613 * in the protocol layers.
2614 *
2615 * Now it will return ENETDOWN, if device have just gone down,
2616 * but then it will block.
2617 */
2618
40d4e3df 2619 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2620
2621 /*
1ce4f28b 2622 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2623 * handles the blocking we don't see and worry about blocking
2624 * retries.
2625 */
2626
8ae55f04 2627 if (skb == NULL)
1da177e4
LT		 2628 goto out;
2629
bfd5f4a3
SS		 2630 if (pkt_sk(sk)->has_vnet_hdr) {
2631 struct virtio_net_hdr vnet_hdr = { 0 };
2632
2633 err = -EINVAL;
2634 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2635 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2636 goto out_free;
2637
1f18b717
MK		 2638 len -= vnet_hdr_len;
2639
bfd5f4a3
SS		 2640 if (skb_is_gso(skb)) {
2641 struct skb_shared_info *sinfo = skb_shinfo(skb);
2642
2643 /* This is a hint as to how much should be linear. */
2644 vnet_hdr.hdr_len = skb_headlen(skb);
2645 vnet_hdr.gso_size = sinfo->gso_size;
2646 if (sinfo->gso_type & SKB_GSO_TCPV4)
2647 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2648 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2649 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2650 else if (sinfo->gso_type & SKB_GSO_UDP)
2651 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2652 else if (sinfo->gso_type & SKB_GSO_FCOE)
2653 goto out_free;
2654 else
2655 BUG();
2656 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2657 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2658 } else
2659 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2660
2661 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2662 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2663 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2664 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2665 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2666 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2667 } /* else everything is zero */
2668
2669 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2670 vnet_hdr_len);
2671 if (err < 0)
2672 goto out_free;
2673 }
2674
0fb375fb
EB		 2675 /*
2676 * If the address length field is there to be filled in, we fill
2677 * it in now.
2678 */
2679
ffbc6111 2680 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2681 if (sock->type == SOCK_PACKET)
2682 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2683 else
2684 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2685
1da177e4
LT		 2686 /*
2687 * You lose any data beyond the buffer you gave. If it worries a
2688 * user program they can ask the device for its MTU anyway.
2689 */
2690
2691 copied = skb->len;
40d4e3df
ED		 2692 if (copied > len) {
2693 copied = len;
2694 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2695 }
2696
2697 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2698 if (err)
2699 goto out_free;
2700
3b885787 2701 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2702
2703 if (msg->msg_name)
ffbc6111
HX		 2704 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2705 msg->msg_namelen);
1da177e4 2706
8dc41944 2707 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2708 struct tpacket_auxdata aux;
2709
2710 aux.tp_status = TP_STATUS_USER;
2711 if (skb->ip_summed == CHECKSUM_PARTIAL)
2712 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2713 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2714 aux.tp_snaplen = skb->len;
2715 aux.tp_mac = 0;
bbe735e4 2716 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2717 if (vlan_tx_tag_present(skb)) {
2718 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2719 aux.tp_status |= TP_STATUS_VLAN_VALID;
2720 } else {
2721 aux.tp_vlan_tci = 0;
2722 }
13fcb7bd 2723 aux.tp_padding = 0;
ffbc6111 2724 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2725 }
2726
1da177e4
LT		 2727 /*
2728 * Free or return the buffer as appropriate. Again this
2729 * hides all the races and re-entrancy issues from us.
2730 */
bfd5f4a3 2731 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2732
2733out_free:
2734 skb_free_datagram(sk, skb);
2735out:
2736 return err;
2737}
2738
1da177e4
LT		 2739static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2740 int *uaddr_len, int peer)
2741{
2742 struct net_device *dev;
2743 struct sock *sk = sock->sk;
2744
2745 if (peer)
2746 return -EOPNOTSUPP;
2747
2748 uaddr->sa_family = AF_PACKET;
654d1f8a
ED		 2749 rcu_read_lock();
2750 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2751 if (dev)
67286640 2752 strncpy(uaddr->sa_data, dev->name, 14);
654d1f8a 2753 else
1da177e4 2754 memset(uaddr->sa_data, 0, 14);
654d1f8a 2755 rcu_read_unlock();
1da177e4
LT		 2756 *uaddr_len = sizeof(*uaddr);
2757
2758 return 0;
2759}
1da177e4
LT		 2760
2761static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2762 int *uaddr_len, int peer)
2763{
2764 struct net_device *dev;
2765 struct sock *sk = sock->sk;
2766 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2767 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2768
2769 if (peer)
2770 return -EOPNOTSUPP;
2771
2772 sll->sll_family = AF_PACKET;
2773 sll->sll_ifindex = po->ifindex;
2774 sll->sll_protocol = po->num;
67286640 2775 sll->sll_pkttype = 0;
654d1f8a
ED		 2776 rcu_read_lock();
2777 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2778 if (dev) {
2779 sll->sll_hatype = dev->type;
2780 sll->sll_halen = dev->addr_len;
2781 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2782 } else {
2783 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2784 sll->sll_halen = 0;
2785 }
654d1f8a 2786 rcu_read_unlock();
0fb375fb 2787 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2788
2789 return 0;
2790}
2791
2aeb0b88
WC		 2792static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2793 int what)
1da177e4
LT		 2794{
2795 switch (i->type) {
2796 case PACKET_MR_MULTICAST:
1162563f
JP		 2797 if (i->alen != dev->addr_len)
2798 return -EINVAL;
1da177e4 2799 if (what > 0)
22bedad3 2800 return dev_mc_add(dev, i->addr);
1da177e4 2801 else
22bedad3 2802 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2803 break;
2804 case PACKET_MR_PROMISC:
2aeb0b88 2805 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2806 break;
2807 case PACKET_MR_ALLMULTI:
2aeb0b88 2808 return dev_set_allmulti(dev, what);
1da177e4 2809 break;
d95ed927 2810 case PACKET_MR_UNICAST:
1162563f
JP		 2811 if (i->alen != dev->addr_len)
2812 return -EINVAL;
d95ed927 2813 if (what > 0)
a748ee24 2814 return dev_uc_add(dev, i->addr);
d95ed927 2815 else
a748ee24 2816 return dev_uc_del(dev, i->addr);
d95ed927 2817 break;
40d4e3df
ED		 2818 default:
2819 break;
1da177e4 2820 }
2aeb0b88 2821 return 0;
1da177e4
LT		 2822}
2823
2824static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2825{
40d4e3df 2826 for ( ; i; i = i->next) {
1da177e4
LT		 2827 if (i->ifindex == dev->ifindex)
2828 packet_dev_mc(dev, i, what);
2829 }
2830}
2831
0fb375fb 2832static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2833{
2834 struct packet_sock *po = pkt_sk(sk);
2835 struct packet_mclist *ml, *i;
2836 struct net_device *dev;
2837 int err;
2838
2839 rtnl_lock();
2840
2841 err = -ENODEV;
3b1e0a65 2842 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2843 if (!dev)
2844 goto done;
2845
2846 err = -EINVAL;
1162563f 2847 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2848 goto done;
2849
2850 err = -ENOBUFS;
8b3a7005 2851 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2852 if (i == NULL)
2853 goto done;
2854
2855 err = 0;
2856 for (ml = po->mclist; ml; ml = ml->next) {
2857 if (ml->ifindex == mreq->mr_ifindex &&
2858 ml->type == mreq->mr_type &&
2859 ml->alen == mreq->mr_alen &&
2860 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2861 ml->count++;
2862 /* Free the new element ... */
2863 kfree(i);
2864 goto done;
2865 }
2866 }
2867
2868 i->type = mreq->mr_type;
2869 i->ifindex = mreq->mr_ifindex;
2870 i->alen = mreq->mr_alen;
2871 memcpy(i->addr, mreq->mr_address, i->alen);
2872 i->count = 1;
2873 i->next = po->mclist;
2874 po->mclist = i;
2aeb0b88
WC		 2875 err = packet_dev_mc(dev, i, 1);
2876 if (err) {
2877 po->mclist = i->next;
2878 kfree(i);
2879 }
1da177e4
LT		 2880
2881done:
2882 rtnl_unlock();
2883 return err;
2884}
2885
0fb375fb 2886static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2887{
2888 struct packet_mclist *ml, **mlp;
2889
2890 rtnl_lock();
2891
2892 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2893 if (ml->ifindex == mreq->mr_ifindex &&
2894 ml->type == mreq->mr_type &&
2895 ml->alen == mreq->mr_alen &&
2896 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2897 if (--ml->count == 0) {
2898 struct net_device *dev;
2899 *mlp = ml->next;
ad959e76
ED		 2900 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2901 if (dev)
1da177e4 2902 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2903 kfree(ml);
2904 }
2905 rtnl_unlock();
2906 return 0;
2907 }
2908 }
2909 rtnl_unlock();
2910 return -EADDRNOTAVAIL;
2911}
2912
2913static void packet_flush_mclist(struct sock *sk)
2914{
2915 struct packet_sock *po = pkt_sk(sk);
2916 struct packet_mclist *ml;
2917
2918 if (!po->mclist)
2919 return;
2920
2921 rtnl_lock();
2922 while ((ml = po->mclist) != NULL) {
2923 struct net_device *dev;
2924
2925 po->mclist = ml->next;
ad959e76
ED		 2926 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2927 if (dev != NULL)
1da177e4 2928 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2929 kfree(ml);
2930 }
2931 rtnl_unlock();
2932}
1da177e4
LT		 2933
2934static int
b7058842 2935packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 2936{
2937 struct sock *sk = sock->sk;
8dc41944 2938 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 2939 int ret;
2940
2941 if (level != SOL_PACKET)
2942 return -ENOPROTOOPT;
2943
69e3c75f 2944 switch (optname) {
1ce4f28b 2945 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 2946 case PACKET_DROP_MEMBERSHIP:
2947 {
0fb375fb
EB		 2948 struct packet_mreq_max mreq;
2949 int len = optlen;
2950 memset(&mreq, 0, sizeof(mreq));
2951 if (len < sizeof(struct packet_mreq))
1da177e4 2952 return -EINVAL;
0fb375fb
EB		 2953 if (len > sizeof(mreq))
2954 len = sizeof(mreq);
40d4e3df 2955 if (copy_from_user(&mreq, optval, len))
1da177e4 2956 return -EFAULT;
0fb375fb
EB		 2957 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
2958 return -EINVAL;
1da177e4
LT		 2959 if (optname == PACKET_ADD_MEMBERSHIP)
2960 ret = packet_mc_add(sk, &mreq);
2961 else
2962 ret = packet_mc_drop(sk, &mreq);
2963 return ret;
2964 }
a2efcfa0 2965
1da177e4 2966 case PACKET_RX_RING:
69e3c75f 2967 case PACKET_TX_RING:
1da177e4 2968 {
f6fb8f10 2969 union tpacket_req_u req_u;
2970 int len;
1da177e4 2971
f6fb8f10 2972 switch (po->tp_version) {
2973 case TPACKET_V1:
2974 case TPACKET_V2:
2975 len = sizeof(req_u.req);
2976 break;
2977 case TPACKET_V3:
2978 default:
2979 len = sizeof(req_u.req3);
2980 break;
2981 }
2982 if (optlen < len)
1da177e4 2983 return -EINVAL;
bfd5f4a3
SS		 2984 if (pkt_sk(sk)->has_vnet_hdr)
2985 return -EINVAL;
f6fb8f10 2986 if (copy_from_user(&req_u.req, optval, len))
1da177e4 2987 return -EFAULT;
f6fb8f10 2988 return packet_set_ring(sk, &req_u, 0,
2989 optname == PACKET_TX_RING);
1da177e4
LT		 2990 }
2991 case PACKET_COPY_THRESH:
2992 {
2993 int val;
2994
40d4e3df 2995 if (optlen != sizeof(val))
1da177e4 2996 return -EINVAL;
40d4e3df 2997 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 2998 return -EFAULT;
2999
3000 pkt_sk(sk)->copy_thresh = val;
3001 return 0;
3002 }
bbd6ef87
PM		 3003 case PACKET_VERSION:
3004 {
3005 int val;
3006
3007 if (optlen != sizeof(val))
3008 return -EINVAL;
69e3c75f 3009 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3010 return -EBUSY;
3011 if (copy_from_user(&val, optval, sizeof(val)))
3012 return -EFAULT;
3013 switch (val) {
3014 case TPACKET_V1:
3015 case TPACKET_V2:
f6fb8f10 3016 case TPACKET_V3:
bbd6ef87
PM		 3017 po->tp_version = val;
3018 return 0;
3019 default:
3020 return -EINVAL;
3021 }
3022 }
8913336a
PM		 3023 case PACKET_RESERVE:
3024 {
3025 unsigned int val;
3026
3027 if (optlen != sizeof(val))
3028 return -EINVAL;
69e3c75f 3029 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3030 return -EBUSY;
3031 if (copy_from_user(&val, optval, sizeof(val)))
3032 return -EFAULT;
3033 po->tp_reserve = val;
3034 return 0;
3035 }
69e3c75f
JB		 3036 case PACKET_LOSS:
3037 {
3038 unsigned int val;
3039
3040 if (optlen != sizeof(val))
3041 return -EINVAL;
3042 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3043 return -EBUSY;
3044 if (copy_from_user(&val, optval, sizeof(val)))
3045 return -EFAULT;
3046 po->tp_loss = !!val;
3047 return 0;
3048 }
8dc41944
HX		 3049 case PACKET_AUXDATA:
3050 {
3051 int val;
3052
3053 if (optlen < sizeof(val))
3054 return -EINVAL;
3055 if (copy_from_user(&val, optval, sizeof(val)))
3056 return -EFAULT;
3057
3058 po->auxdata = !!val;
3059 return 0;
3060 }
80feaacb
PWJ		 3061 case PACKET_ORIGDEV:
3062 {
3063 int val;
3064
3065 if (optlen < sizeof(val))
3066 return -EINVAL;
3067 if (copy_from_user(&val, optval, sizeof(val)))
3068 return -EFAULT;
3069
3070 po->origdev = !!val;
3071 return 0;
3072 }
bfd5f4a3
SS		 3073 case PACKET_VNET_HDR:
3074 {
3075 int val;
3076
3077 if (sock->type != SOCK_RAW)
3078 return -EINVAL;
3079 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3080 return -EBUSY;
3081 if (optlen < sizeof(val))
3082 return -EINVAL;
3083 if (copy_from_user(&val, optval, sizeof(val)))
3084 return -EFAULT;
3085
3086 po->has_vnet_hdr = !!val;
3087 return 0;
3088 }
614f60fa
SM		 3089 case PACKET_TIMESTAMP:
3090 {
3091 int val;
3092
3093 if (optlen != sizeof(val))
3094 return -EINVAL;
3095 if (copy_from_user(&val, optval, sizeof(val)))
3096 return -EFAULT;
3097
3098 po->tp_tstamp = val;
3099 return 0;
3100 }
dc99f600
DM		 3101 case PACKET_FANOUT:
3102 {
3103 int val;
3104
3105 if (optlen != sizeof(val))
3106 return -EINVAL;
3107 if (copy_from_user(&val, optval, sizeof(val)))
3108 return -EFAULT;
3109
3110 return fanout_add(sk, val & 0xffff, val >> 16);
3111 }
1da177e4
LT		 3112 default:
3113 return -ENOPROTOOPT;
3114 }
3115}
3116
3117static int packet_getsockopt(struct socket *sock, int level, int optname,
3118 char __user *optval, int __user *optlen)
3119{
3120 int len;
c06fff6e 3121 int val, lv = sizeof(val);
1da177e4
LT		 3122 struct sock *sk = sock->sk;
3123 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3124 void *data = &val;
8dc41944 3125 struct tpacket_stats st;
f6fb8f10 3126 union tpacket_stats_u st_u;
1da177e4
LT		 3127
3128 if (level != SOL_PACKET)
3129 return -ENOPROTOOPT;
3130
8ae55f04
KK		 3131 if (get_user(len, optlen))
3132 return -EFAULT;
1da177e4
LT		 3133
3134 if (len < 0)
3135 return -EINVAL;
1ce4f28b 3136
69e3c75f 3137 switch (optname) {
1da177e4 3138 case PACKET_STATISTICS:
1da177e4 3139 spin_lock_bh(&sk->sk_receive_queue.lock);
f6fb8f10 3140 if (po->tp_version == TPACKET_V3) {
c06fff6e 3141 lv = sizeof(struct tpacket_stats_v3);
f6fb8f10 3142 memcpy(&st_u.stats3, &po->stats,
c06fff6e 3143 sizeof(struct tpacket_stats));
f6fb8f10 3144 st_u.stats3.tp_freeze_q_cnt =
c06fff6e 3145 po->stats_u.stats3.tp_freeze_q_cnt;
f6fb8f10 3146 st_u.stats3.tp_packets += po->stats.tp_drops;
3147 data = &st_u.stats3;
3148 } else {
c06fff6e 3149 lv = sizeof(struct tpacket_stats);
f6fb8f10 3150 st = po->stats;
3151 st.tp_packets += st.tp_drops;
3152 data = &st;
3153 }
1da177e4
LT		 3154 memset(&po->stats, 0, sizeof(st));
3155 spin_unlock_bh(&sk->sk_receive_queue.lock);
8dc41944
HX		 3156 break;
3157 case PACKET_AUXDATA:
8dc41944 3158 val = po->auxdata;
80feaacb
PWJ		 3159 break;
3160 case PACKET_ORIGDEV:
80feaacb 3161 val = po->origdev;
bfd5f4a3
SS		 3162 break;
3163 case PACKET_VNET_HDR:
bfd5f4a3 3164 val = po->has_vnet_hdr;
1da177e4 3165 break;
bbd6ef87 3166 case PACKET_VERSION:
bbd6ef87 3167 val = po->tp_version;
bbd6ef87
PM		 3168 break;
3169 case PACKET_HDRLEN:
3170 if (len > sizeof(int))
3171 len = sizeof(int);
3172 if (copy_from_user(&val, optval, len))
3173 return -EFAULT;
3174 switch (val) {
3175 case TPACKET_V1:
3176 val = sizeof(struct tpacket_hdr);
3177 break;
3178 case TPACKET_V2:
3179 val = sizeof(struct tpacket2_hdr);
3180 break;
f6fb8f10 3181 case TPACKET_V3:
3182 val = sizeof(struct tpacket3_hdr);
3183 break;
bbd6ef87
PM		 3184 default:
3185 return -EINVAL;
3186 }
bbd6ef87 3187 break;
8913336a 3188 case PACKET_RESERVE:
8913336a 3189 val = po->tp_reserve;
8913336a 3190 break;
69e3c75f 3191 case PACKET_LOSS:
69e3c75f 3192 val = po->tp_loss;
69e3c75f 3193 break;
614f60fa 3194 case PACKET_TIMESTAMP:
614f60fa 3195 val = po->tp_tstamp;
614f60fa 3196 break;
dc99f600 3197 case PACKET_FANOUT:
dc99f600
DM		 3198 val = (po->fanout ?
3199 ((u32)po->fanout->id |
3200 ((u32)po->fanout->type << 16)) :
3201 0);
dc99f600 3202 break;
1da177e4
LT		 3203 default:
3204 return -ENOPROTOOPT;
3205 }
3206
c06fff6e
ED		 3207 if (len > lv)
3208 len = lv;
8ae55f04
KK		 3209 if (put_user(len, optlen))
3210 return -EFAULT;
8dc41944
HX		 3211 if (copy_to_user(optval, data, len))
3212 return -EFAULT;
8ae55f04 3213 return 0;
1da177e4
LT		 3214}
3215
3216
3217static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3218{
3219 struct sock *sk;
3220 struct hlist_node *node;
ad930650 3221 struct net_device *dev = data;
c346dca1 3222 struct net *net = dev_net(dev);
1da177e4 3223
808f5114 3224 rcu_read_lock();
3225 sk_for_each_rcu(sk, node, &net->packet.sklist) {
1da177e4
LT		 3226 struct packet_sock *po = pkt_sk(sk);
3227
3228 switch (msg) {
3229 case NETDEV_UNREGISTER:
1da177e4
LT		 3230 if (po->mclist)
3231 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3232 /* fallthrough */
3233
1da177e4
LT		 3234 case NETDEV_DOWN:
3235 if (dev->ifindex == po->ifindex) {
3236 spin_lock(&po->bind_lock);
3237 if (po->running) {
ce06b03e 3238 __unregister_prot_hook(sk, false);
1da177e4
LT		 3239 sk->sk_err = ENETDOWN;
3240 if (!sock_flag(sk, SOCK_DEAD))
3241 sk->sk_error_report(sk);
3242 }
3243 if (msg == NETDEV_UNREGISTER) {
3244 po->ifindex = -1;
160ff18a
BG		 3245 if (po->prot_hook.dev)
3246 dev_put(po->prot_hook.dev);
1da177e4
LT		 3247 po->prot_hook.dev = NULL;
3248 }
3249 spin_unlock(&po->bind_lock);
3250 }
3251 break;
3252 case NETDEV_UP:
808f5114 3253 if (dev->ifindex == po->ifindex) {
3254 spin_lock(&po->bind_lock);
ce06b03e
DM		 3255 if (po->num)
3256 register_prot_hook(sk);
808f5114 3257 spin_unlock(&po->bind_lock);
1da177e4 3258 }
1da177e4
LT		 3259 break;
3260 }
3261 }
808f5114 3262 rcu_read_unlock();
1da177e4
LT		 3263 return NOTIFY_DONE;
3264}
3265
3266
3267static int packet_ioctl(struct socket *sock, unsigned int cmd,
3268 unsigned long arg)
3269{
3270 struct sock *sk = sock->sk;
3271
69e3c75f 3272 switch (cmd) {
40d4e3df
ED		 3273 case SIOCOUTQ:
3274 {
3275 int amount = sk_wmem_alloc_get(sk);
31e6d363 3276
40d4e3df
ED		 3277 return put_user(amount, (int __user *)arg);
3278 }
3279 case SIOCINQ:
3280 {
3281 struct sk_buff *skb;
3282 int amount = 0;
3283
3284 spin_lock_bh(&sk->sk_receive_queue.lock);
3285 skb = skb_peek(&sk->sk_receive_queue);
3286 if (skb)
3287 amount = skb->len;
3288 spin_unlock_bh(&sk->sk_receive_queue.lock);
3289 return put_user(amount, (int __user *)arg);
3290 }
3291 case SIOCGSTAMP:
3292 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3293 case SIOCGSTAMPNS:
3294 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3295
1da177e4 3296#ifdef CONFIG_INET
40d4e3df
ED		 3297 case SIOCADDRT:
3298 case SIOCDELRT:
3299 case SIOCDARP:
3300 case SIOCGARP:
3301 case SIOCSARP:
3302 case SIOCGIFADDR:
3303 case SIOCSIFADDR:
3304 case SIOCGIFBRDADDR:
3305 case SIOCSIFBRDADDR:
3306 case SIOCGIFNETMASK:
3307 case SIOCSIFNETMASK:
3308 case SIOCGIFDSTADDR:
3309 case SIOCSIFDSTADDR:
3310 case SIOCSIFFLAGS:
40d4e3df 3311 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3312#endif
3313
40d4e3df
ED		 3314 default:
3315 return -ENOIOCTLCMD;
1da177e4
LT		 3316 }
3317 return 0;
3318}
3319
40d4e3df 3320static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3321 poll_table *wait)
3322{
3323 struct sock *sk = sock->sk;
3324 struct packet_sock *po = pkt_sk(sk);
3325 unsigned int mask = datagram_poll(file, sock, wait);
3326
3327 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3328 if (po->rx_ring.pg_vec) {
f6fb8f10 3329 if (!packet_previous_rx_frame(po, &po->rx_ring,
3330 TP_STATUS_KERNEL))
1da177e4
LT		 3331 mask |= POLLIN | POLLRDNORM;
3332 }
3333 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3334 spin_lock_bh(&sk->sk_write_queue.lock);
3335 if (po->tx_ring.pg_vec) {
3336 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3337 mask |= POLLOUT | POLLWRNORM;
3338 }
3339 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3340 return mask;
3341}
3342
3343
3344/* Dirty? Well, I still did not learn better way to account
3345 * for user mmaps.
3346 */
3347
3348static void packet_mm_open(struct vm_area_struct *vma)
3349{
3350 struct file *file = vma->vm_file;
40d4e3df 3351 struct socket *sock = file->private_data;
1da177e4 3352 struct sock *sk = sock->sk;
1ce4f28b 3353
1da177e4
LT		 3354 if (sk)
3355 atomic_inc(&pkt_sk(sk)->mapped);
3356}
3357
3358static void packet_mm_close(struct vm_area_struct *vma)
3359{
3360 struct file *file = vma->vm_file;
40d4e3df 3361 struct socket *sock = file->private_data;
1da177e4 3362 struct sock *sk = sock->sk;
1ce4f28b 3363
1da177e4
LT		 3364 if (sk)
3365 atomic_dec(&pkt_sk(sk)->mapped);
3366}
3367
f0f37e2f 3368static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3369 .open = packet_mm_open,
3370 .close = packet_mm_close,
1da177e4
LT		 3371};
3372
0e3125c7
NH		 3373static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3374 unsigned int len)
1da177e4
LT		 3375{
3376 int i;
3377
4ebf0ae2 3378 for (i = 0; i < len; i++) {
0e3125c7 3379 if (likely(pg_vec[i].buffer)) {
c56b4d90 3380 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3381 vfree(pg_vec[i].buffer);
3382 else
3383 free_pages((unsigned long)pg_vec[i].buffer,
3384 order);
3385 pg_vec[i].buffer = NULL;
3386 }
1da177e4
LT		 3387 }
3388 kfree(pg_vec);
3389}
3390
eea49cc9 3391static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3392{
0e3125c7
NH		 3393 char *buffer = NULL;
3394 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3395 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3396
3397 buffer = (char *) __get_free_pages(gfp_flags, order);
3398
3399 if (buffer)
3400 return buffer;
3401
3402 /*
3403 * __get_free_pages failed, fall back to vmalloc
3404 */
bbce5a59 3405 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3406
0e3125c7
NH		 3407 if (buffer)
3408 return buffer;
3409
3410 /*
3411 * vmalloc failed, lets dig into swap here
3412 */
0e3125c7
NH		 3413 gfp_flags &= ~__GFP_NORETRY;
3414 buffer = (char *)__get_free_pages(gfp_flags, order);
3415 if (buffer)
3416 return buffer;
3417
3418 /*
3419 * complete and utter failure
3420 */
3421 return NULL;
4ebf0ae2
DM		 3422}
3423
0e3125c7 3424static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3425{
3426 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3427 struct pgv *pg_vec;
4ebf0ae2
DM		 3428 int i;
3429
0e3125c7 3430 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3431 if (unlikely(!pg_vec))
3432 goto out;
3433
3434 for (i = 0; i < block_nr; i++) {
c56b4d90 3435 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3436 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3437 goto out_free_pgvec;
3438 }
3439
3440out:
3441 return pg_vec;
3442
3443out_free_pgvec:
3444 free_pg_vec(pg_vec, order, block_nr);
3445 pg_vec = NULL;
3446 goto out;
3447}
1da177e4 3448
f6fb8f10 3449static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3450 int closing, int tx_ring)
1da177e4 3451{
0e3125c7 3452 struct pgv *pg_vec = NULL;
1da177e4 3453 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3454 int was_running, order = 0;
69e3c75f
JB		 3455 struct packet_ring_buffer *rb;
3456 struct sk_buff_head *rb_queue;
0e11c91e 3457 __be16 num;
f6fb8f10 3458 int err = -EINVAL;
3459 /* Added to avoid minimal code churn */
3460 struct tpacket_req *req = &req_u->req;
3461
3462 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3463 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3464 WARN(1, "Tx-ring is not supported.\n");
3465 goto out;
3466 }
1ce4f28b 3467
69e3c75f
JB		 3468 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3469 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3470
69e3c75f
JB		 3471 err = -EBUSY;
3472 if (!closing) {
3473 if (atomic_read(&po->mapped))
3474 goto out;
3475 if (atomic_read(&rb->pending))
3476 goto out;
3477 }
1da177e4 3478
69e3c75f
JB		 3479 if (req->tp_block_nr) {
3480 /* Sanity tests and some calculations */
3481 err = -EBUSY;
3482 if (unlikely(rb->pg_vec))
3483 goto out;
1da177e4 3484
bbd6ef87
PM		 3485 switch (po->tp_version) {
3486 case TPACKET_V1:
3487 po->tp_hdrlen = TPACKET_HDRLEN;
3488 break;
3489 case TPACKET_V2:
3490 po->tp_hdrlen = TPACKET2_HDRLEN;
3491 break;
f6fb8f10 3492 case TPACKET_V3:
3493 po->tp_hdrlen = TPACKET3_HDRLEN;
3494 break;
bbd6ef87
PM		 3495 }
3496
69e3c75f 3497 err = -EINVAL;
4ebf0ae2 3498 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3499 goto out;
4ebf0ae2 3500 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3501 goto out;
8913336a 3502 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3503 po->tp_reserve))
3504 goto out;
4ebf0ae2 3505 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3506 goto out;
1da177e4 3507
69e3c75f
JB		 3508 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3509 if (unlikely(rb->frames_per_block <= 0))
3510 goto out;
3511 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3512 req->tp_frame_nr))
3513 goto out;
1da177e4
LT		 3514
3515 err = -ENOMEM;
4ebf0ae2
DM		 3516 order = get_order(req->tp_block_size);
3517 pg_vec = alloc_pg_vec(req, order);
3518 if (unlikely(!pg_vec))
1da177e4 3519 goto out;
f6fb8f10 3520 switch (po->tp_version) {
3521 case TPACKET_V3:
3522 /* Transmit path is not supported. We checked
3523 * it above but just being paranoid
3524 */
3525 if (!tx_ring)
3526 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3527 break;
3528 default:
3529 break;
3530 }
69e3c75f
JB		 3531 }
3532 /* Done */
3533 else {
3534 err = -EINVAL;
4ebf0ae2 3535 if (unlikely(req->tp_frame_nr))
69e3c75f 3536 goto out;
1da177e4
LT		 3537 }
3538
3539 lock_sock(sk);
3540
3541 /* Detach socket from network */
3542 spin_lock(&po->bind_lock);
3543 was_running = po->running;
3544 num = po->num;
3545 if (was_running) {
1da177e4 3546 po->num = 0;
ce06b03e 3547 __unregister_prot_hook(sk, false);
1da177e4
LT		 3548 }
3549 spin_unlock(&po->bind_lock);
1ce4f28b 3550
1da177e4
LT		 3551 synchronize_net();
3552
3553 err = -EBUSY;
905db440 3554 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3555 if (closing || atomic_read(&po->mapped) == 0) {
3556 err = 0;
69e3c75f 3557 spin_lock_bh(&rb_queue->lock);
c053fd96 3558 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3559 rb->frame_max = (req->tp_frame_nr - 1);
3560 rb->head = 0;
3561 rb->frame_size = req->tp_frame_size;
3562 spin_unlock_bh(&rb_queue->lock);
3563
c053fd96
CG		 3564 swap(rb->pg_vec_order, order);
3565 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3566
3567 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3568 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3569 tpacket_rcv : packet_rcv;
3570 skb_queue_purge(rb_queue);
1da177e4 3571 if (atomic_read(&po->mapped))
40d4e3df
ED		 3572 pr_err("packet_mmap: vma is busy: %d\n",
3573 atomic_read(&po->mapped));
1da177e4 3574 }
905db440 3575 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3576
3577 spin_lock(&po->bind_lock);
ce06b03e 3578 if (was_running) {
1da177e4 3579 po->num = num;
ce06b03e 3580 register_prot_hook(sk);
1da177e4
LT		 3581 }
3582 spin_unlock(&po->bind_lock);
f6fb8f10 3583 if (closing && (po->tp_version > TPACKET_V2)) {
3584 /* Because we don't support block-based V3 on tx-ring */
3585 if (!tx_ring)
3586 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3587 }
1da177e4
LT		 3588 release_sock(sk);
3589
1da177e4
LT		 3590 if (pg_vec)
3591 free_pg_vec(pg_vec, order, req->tp_block_nr);
3592out:
3593 return err;
3594}
3595
69e3c75f
JB		 3596static int packet_mmap(struct file *file, struct socket *sock,
3597 struct vm_area_struct *vma)
1da177e4
LT		 3598{
3599 struct sock *sk = sock->sk;
3600 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3601 unsigned long size, expected_size;
3602 struct packet_ring_buffer *rb;
1da177e4
LT		 3603 unsigned long start;
3604 int err = -EINVAL;
3605 int i;
3606
3607 if (vma->vm_pgoff)
3608 return -EINVAL;
3609
905db440 3610 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3611
3612 expected_size = 0;
3613 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3614 if (rb->pg_vec) {
3615 expected_size += rb->pg_vec_len
3616 * rb->pg_vec_pages
3617 * PAGE_SIZE;
3618 }
3619 }
3620
3621 if (expected_size == 0)
1da177e4 3622 goto out;
69e3c75f
JB		 3623
3624 size = vma->vm_end - vma->vm_start;
3625 if (size != expected_size)
1da177e4
LT		 3626 goto out;
3627
1da177e4 3628 start = vma->vm_start;
69e3c75f
JB		 3629 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3630 if (rb->pg_vec == NULL)
3631 continue;
3632
3633 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3634 struct page *page;
3635 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3636 int pg_num;
3637
c56b4d90
CG		 3638 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3639 page = pgv_to_page(kaddr);
69e3c75f
JB		 3640 err = vm_insert_page(vma, start, page);
3641 if (unlikely(err))
3642 goto out;
3643 start += PAGE_SIZE;
0e3125c7 3644 kaddr += PAGE_SIZE;
69e3c75f 3645 }
4ebf0ae2 3646 }
1da177e4 3647 }
69e3c75f 3648
4ebf0ae2 3649 atomic_inc(&po->mapped);
1da177e4
LT		 3650 vma->vm_ops = &packet_mmap_ops;
3651 err = 0;
3652
3653out:
905db440 3654 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3655 return err;
3656}
1da177e4 3657
90ddc4f0 3658static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3659 .family = PF_PACKET,
3660 .owner = THIS_MODULE,
3661 .release = packet_release,
3662 .bind = packet_bind_spkt,
3663 .connect = sock_no_connect,
3664 .socketpair = sock_no_socketpair,
3665 .accept = sock_no_accept,
3666 .getname = packet_getname_spkt,
3667 .poll = datagram_poll,
3668 .ioctl = packet_ioctl,
3669 .listen = sock_no_listen,
3670 .shutdown = sock_no_shutdown,
3671 .setsockopt = sock_no_setsockopt,
3672 .getsockopt = sock_no_getsockopt,
3673 .sendmsg = packet_sendmsg_spkt,
3674 .recvmsg = packet_recvmsg,
3675 .mmap = sock_no_mmap,
3676 .sendpage = sock_no_sendpage,
3677};
1da177e4 3678
90ddc4f0 3679static const struct proto_ops packet_ops = {
1da177e4
LT		 3680 .family = PF_PACKET,
3681 .owner = THIS_MODULE,
3682 .release = packet_release,
3683 .bind = packet_bind,
3684 .connect = sock_no_connect,
3685 .socketpair = sock_no_socketpair,
3686 .accept = sock_no_accept,
1ce4f28b 3687 .getname = packet_getname,
1da177e4
LT		 3688 .poll = packet_poll,
3689 .ioctl = packet_ioctl,
3690 .listen = sock_no_listen,
3691 .shutdown = sock_no_shutdown,
3692 .setsockopt = packet_setsockopt,
3693 .getsockopt = packet_getsockopt,
3694 .sendmsg = packet_sendmsg,
3695 .recvmsg = packet_recvmsg,
3696 .mmap = packet_mmap,
3697 .sendpage = sock_no_sendpage,
3698};
3699
ec1b4cf7 3700static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3701 .family = PF_PACKET,
3702 .create = packet_create,
3703 .owner = THIS_MODULE,
3704};
3705
3706static struct notifier_block packet_netdev_notifier = {
40d4e3df 3707 .notifier_call = packet_notifier,
1da177e4
LT		 3708};
3709
3710#ifdef CONFIG_PROC_FS
1da177e4
LT		 3711
3712static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3713 __acquires(RCU)
1da177e4 3714{
e372c414 3715 struct net *net = seq_file_net(seq);
808f5114 3716
3717 rcu_read_lock();
3718 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3719}
3720
3721static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3722{
1bf40954 3723 struct net *net = seq_file_net(seq);
808f5114 3724 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3725}
3726
3727static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3728 __releases(RCU)
1da177e4 3729{
808f5114 3730 rcu_read_unlock();
1da177e4
LT		 3731}
3732
1ce4f28b 3733static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3734{
3735 if (v == SEQ_START_TOKEN)
3736 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3737 else {
b7ceabd9 3738 struct sock *s = sk_entry(v);
1da177e4
LT		 3739 const struct packet_sock *po = pkt_sk(s);
3740
3741 seq_printf(seq,
71338aa7 3742 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3743 s,
3744 atomic_read(&s->sk_refcnt),
3745 s->sk_type,
3746 ntohs(po->num),
3747 po->ifindex,
3748 po->running,
3749 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3750 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3751 sock_i_ino(s));
1da177e4
LT		 3752 }
3753
3754 return 0;
3755}
3756
56b3d975 3757static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3758 .start = packet_seq_start,
3759 .next = packet_seq_next,
3760 .stop = packet_seq_stop,
3761 .show = packet_seq_show,
3762};
3763
3764static int packet_seq_open(struct inode *inode, struct file *file)
3765{
e372c414
DL		 3766 return seq_open_net(inode, file, &packet_seq_ops,
3767 sizeof(struct seq_net_private));
1da177e4
LT		 3768}
3769
da7071d7 3770static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3771 .owner = THIS_MODULE,
3772 .open = packet_seq_open,
3773 .read = seq_read,
3774 .llseek = seq_lseek,
e372c414 3775 .release = seq_release_net,
1da177e4
LT		 3776};
3777
3778#endif
3779
2c8c1e72 3780static int __net_init packet_net_init(struct net *net)
d12d01d6 3781{
0fa7fa98 3782 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3783 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6
DL		 3784
3785 if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
3786 return -ENOMEM;
3787
3788 return 0;
3789}
3790
2c8c1e72 3791static void __net_exit packet_net_exit(struct net *net)
d12d01d6
DL		 3792{
3793 proc_net_remove(net, "packet");
3794}
3795
3796static struct pernet_operations packet_net_ops = {
3797 .init = packet_net_init,
3798 .exit = packet_net_exit,
3799};
3800
3801
1da177e4
LT		 3802static void __exit packet_exit(void)
3803{
1da177e4 3804 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3805 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3806 sock_unregister(PF_PACKET);
3807 proto_unregister(&packet_proto);
3808}
3809
3810static int __init packet_init(void)
3811{
3812 int rc = proto_register(&packet_proto, 0);
3813
3814 if (rc != 0)
3815 goto out;
3816
3817 sock_register(&packet_family_ops);
d12d01d6 3818 register_pernet_subsys(&packet_net_ops);
1da177e4 3819 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3820out:
3821 return rc;
3822}
3823
3824module_init(packet_init);
3825module_exit(packet_exit);
3826MODULE_LICENSE("GPL");
3827MODULE_ALIAS_NETPROTO(PF_PACKET);
ubuntu-zesty-kernel mirror