]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - net/packet/af_packet.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
lwtunnel: rename ip lwtunnel attributes
[mirror_ubuntu-zesty-kernel.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111 218struct packet_skb_cb {
ffbc6111
HX		 219 union {
220 struct sockaddr_pkt pkt;
2472d761
EB		 221 union {
222 /* Trick: alias skb original length with
223 * ll.sll_family and ll.protocol in order
224 * to save room.
225 */
226 unsigned int origlen;
227 struct sockaddr_ll ll;
228 };
ffbc6111
HX		 229 } sa;
230};
231
232#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 233
bc59ba39 234#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 235#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 236 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 237#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 238 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 239#define GET_NEXT_PRB_BLK_NUM(x) \
240 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
241 ((x)->kactive_blk_num+1) : 0)
242
dc99f600
DM		 243static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244static void __fanout_link(struct sock *sk, struct packet_sock *po);
245
d346a3fa
DB		 246static int packet_direct_xmit(struct sk_buff *skb)
247{
248 struct net_device *dev = skb->dev;
d346a3fa
DB		 249 netdev_features_t features;
250 struct netdev_queue *txq;
43279500 251 int ret = NETDEV_TX_BUSY;
d346a3fa
DB		 252
253 if (unlikely(!netif_running(dev) ||
43279500
DB		 254 !netif_carrier_ok(dev)))
255 goto drop;
d346a3fa
DB		 256
257 features = netif_skb_features(skb);
258 if (skb_needs_linearize(skb, features) &&
43279500
DB		 259 __skb_linearize(skb))
260 goto drop;
d346a3fa 261
10c51b56 262 txq = skb_get_tx_queue(dev, skb);
d346a3fa 263
43279500
DB		 264 local_bh_disable();
265
266 HARD_TX_LOCK(dev, txq, smp_processor_id());
10b3ad8c 267 if (!netif_xmit_frozen_or_drv_stopped(txq))
fa2dbdc2 268 ret = netdev_start_xmit(skb, dev, txq, false);
43279500 269 HARD_TX_UNLOCK(dev, txq);
d346a3fa 270
43279500
DB		 271 local_bh_enable();
272
273 if (!dev_xmit_complete(ret))
d346a3fa 274 kfree_skb(skb);
43279500 275
d346a3fa 276 return ret;
43279500 277drop:
0f97ede4 278 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 279 kfree_skb(skb);
280 return NET_XMIT_DROP;
d346a3fa
DB		 281}
282
66e56cd4
DB		 283static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284{
285 struct net_device *dev;
286
287 rcu_read_lock();
288 dev = rcu_dereference(po->cached_dev);
289 if (likely(dev))
290 dev_hold(dev);
291 rcu_read_unlock();
292
293 return dev;
294}
295
296static void packet_cached_dev_assign(struct packet_sock *po,
297 struct net_device *dev)
298{
299 rcu_assign_pointer(po->cached_dev, dev);
300}
301
302static void packet_cached_dev_reset(struct packet_sock *po)
303{
304 RCU_INIT_POINTER(po->cached_dev, NULL);
305}
306
d346a3fa
DB		 307static bool packet_use_direct_xmit(const struct packet_sock *po)
308{
309 return po->xmit == packet_direct_xmit;
310}
311
0fd5d57b 312static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 313{
1cbac010 314 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 315}
316
0fd5d57b
DB		 317static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
318{
319 const struct net_device_ops *ops = dev->netdev_ops;
320 u16 queue_index;
321
322 if (ops->ndo_select_queue) {
323 queue_index = ops->ndo_select_queue(dev, skb, NULL,
324 __packet_pick_tx_queue);
325 queue_index = netdev_cap_txqueue(dev, queue_index);
326 } else {
327 queue_index = __packet_pick_tx_queue(dev, skb);
328 }
329
330 skb_set_queue_mapping(skb, queue_index);
331}
332
ce06b03e
DM		 333/* register_prot_hook must be invoked with the po->bind_lock held,
334 * or from a context in which asynchronous accesses to the packet
335 * socket is not possible (packet_create()).
336 */
337static void register_prot_hook(struct sock *sk)
338{
339 struct packet_sock *po = pkt_sk(sk);
e40526cb 340
ce06b03e 341 if (!po->running) {
66e56cd4 342 if (po->fanout)
dc99f600 343 __fanout_link(sk, po);
66e56cd4 344 else
dc99f600 345 dev_add_pack(&po->prot_hook);
e40526cb 346
ce06b03e
DM		 347 sock_hold(sk);
348 po->running = 1;
349 }
350}
351
352/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
353 * held. If the sync parameter is true, we will temporarily drop
354 * the po->bind_lock and do a synchronize_net to make sure no
355 * asynchronous packet processing paths still refer to the elements
356 * of po->prot_hook. If the sync parameter is false, it is the
357 * callers responsibility to take care of this.
358 */
359static void __unregister_prot_hook(struct sock *sk, bool sync)
360{
361 struct packet_sock *po = pkt_sk(sk);
362
363 po->running = 0;
66e56cd4
DB		 364
365 if (po->fanout)
dc99f600 366 __fanout_unlink(sk, po);
66e56cd4 367 else
dc99f600 368 __dev_remove_pack(&po->prot_hook);
e40526cb 369
ce06b03e
DM		 370 __sock_put(sk);
371
372 if (sync) {
373 spin_unlock(&po->bind_lock);
374 synchronize_net();
375 spin_lock(&po->bind_lock);
376 }
377}
378
379static void unregister_prot_hook(struct sock *sk, bool sync)
380{
381 struct packet_sock *po = pkt_sk(sk);
382
383 if (po->running)
384 __unregister_prot_hook(sk, sync);
385}
386
6e58040b 387static inline struct page * __pure pgv_to_page(void *addr)
0af55bb5
CG		 388{
389 if (is_vmalloc_addr(addr))
390 return vmalloc_to_page(addr);
391 return virt_to_page(addr);
392}
393
69e3c75f 394static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 395{
184f489e 396 union tpacket_uhdr h;
1da177e4 397
69e3c75f 398 h.raw = frame;
bbd6ef87
PM		 399 switch (po->tp_version) {
400 case TPACKET_V1:
69e3c75f 401 h.h1->tp_status = status;
0af55bb5 402 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 403 break;
404 case TPACKET_V2:
69e3c75f 405 h.h2->tp_status = status;
0af55bb5 406 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 407 break;
f6fb8f10 408 case TPACKET_V3:
69e3c75f 409 default:
f6fb8f10 410 WARN(1, "TPACKET version not supported.\n");
69e3c75f 411 BUG();
bbd6ef87 412 }
69e3c75f
JB		 413
414 smp_wmb();
bbd6ef87
PM		 415}
416
69e3c75f 417static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 418{
184f489e 419 union tpacket_uhdr h;
bbd6ef87 420
69e3c75f
JB		 421 smp_rmb();
422
bbd6ef87
PM		 423 h.raw = frame;
424 switch (po->tp_version) {
425 case TPACKET_V1:
0af55bb5 426 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 427 return h.h1->tp_status;
bbd6ef87 428 case TPACKET_V2:
0af55bb5 429 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 430 return h.h2->tp_status;
f6fb8f10 431 case TPACKET_V3:
69e3c75f 432 default:
f6fb8f10 433 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 434 BUG();
435 return 0;
bbd6ef87 436 }
1da177e4 437}
69e3c75f 438
b9c32fb2
DB		 439static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
440 unsigned int flags)
7a51384c
DB		 441{
442 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
443
68a360e8
WB		 444 if (shhwtstamps &&
445 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
446 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
447 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 448
449 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 450 return TP_STATUS_TS_SOFTWARE;
7a51384c 451
b9c32fb2 452 return 0;
7a51384c
DB		 453}
454
b9c32fb2
DB		 455static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
456 struct sk_buff *skb)
2e31396f
WB		 457{
458 union tpacket_uhdr h;
459 struct timespec ts;
b9c32fb2 460 __u32 ts_status;
2e31396f 461
b9c32fb2
DB		 462 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
463 return 0;
2e31396f
WB		 464
465 h.raw = frame;
466 switch (po->tp_version) {
467 case TPACKET_V1:
468 h.h1->tp_sec = ts.tv_sec;
469 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
470 break;
471 case TPACKET_V2:
472 h.h2->tp_sec = ts.tv_sec;
473 h.h2->tp_nsec = ts.tv_nsec;
474 break;
475 case TPACKET_V3:
476 default:
477 WARN(1, "TPACKET version not supported.\n");
478 BUG();
479 }
480
481 /* one flush is safe, as both fields always lie on the same cacheline */
482 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
483 smp_wmb();
b9c32fb2
DB		 484
485 return ts_status;
2e31396f
WB		 486}
487
69e3c75f
JB		 488static void *packet_lookup_frame(struct packet_sock *po,
489 struct packet_ring_buffer *rb,
490 unsigned int position,
491 int status)
492{
493 unsigned int pg_vec_pos, frame_offset;
184f489e 494 union tpacket_uhdr h;
69e3c75f
JB		 495
496 pg_vec_pos = position / rb->frames_per_block;
497 frame_offset = position % rb->frames_per_block;
498
0e3125c7
NH		 499 h.raw = rb->pg_vec[pg_vec_pos].buffer +
500 (frame_offset * rb->frame_size);
69e3c75f
JB		 501
502 if (status != __packet_get_status(po, h.raw))
503 return NULL;
504
505 return h.raw;
506}
507
eea49cc9 508static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 509 struct packet_ring_buffer *rb,
510 int status)
511{
512 return packet_lookup_frame(po, rb, rb->head, status);
513}
514
bc59ba39 515static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 516{
517 del_timer_sync(&pkc->retire_blk_timer);
518}
519
520static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
f6fb8f10 521 struct sk_buff_head *rb_queue)
522{
bc59ba39 523 struct tpacket_kbdq_core *pkc;
f6fb8f10 524
73d0fcf2 525 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 526
ec6f809f 527 spin_lock_bh(&rb_queue->lock);
f6fb8f10 528 pkc->delete_blk_timer = 1;
ec6f809f 529 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 530
531 prb_del_retire_blk_timer(pkc);
532}
533
534static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 535 struct tpacket_kbdq_core *pkc,
f6fb8f10 536 void (*func) (unsigned long))
537{
538 init_timer(&pkc->retire_blk_timer);
539 pkc->retire_blk_timer.data = (long)po;
540 pkc->retire_blk_timer.function = func;
541 pkc->retire_blk_timer.expires = jiffies;
542}
543
e8e85cc5 544static void prb_setup_retire_blk_timer(struct packet_sock *po)
f6fb8f10 545{
bc59ba39 546 struct tpacket_kbdq_core *pkc;
f6fb8f10 547
e8e85cc5 548 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 549 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
550}
551
552static int prb_calc_retire_blk_tmo(struct packet_sock *po,
553 int blk_size_in_bytes)
554{
555 struct net_device *dev;
556 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 557 struct ethtool_cmd ecmd;
558 int err;
e440cf2c 559 u32 speed;
f6fb8f10 560
4bc71cb9
JP		 561 rtnl_lock();
562 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
563 if (unlikely(!dev)) {
564 rtnl_unlock();
f6fb8f10 565 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 566 }
567 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 568 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 569 rtnl_unlock();
570 if (!err) {
4bc71cb9
JP		 571 /*
572 * If the link speed is so slow you don't really
573 * need to worry about perf anyways
574 */
e440cf2c 575 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 576 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 577 } else {
578 msec = 1;
579 div = speed / 1000;
f6fb8f10 580 }
581 }
582
583 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
584
585 if (div)
586 mbits /= div;
587
588 tmo = mbits * msec;
589
590 if (div)
591 return tmo+1;
592 return tmo;
593}
594
bc59ba39 595static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 596 union tpacket_req_u *req_u)
597{
598 p1->feature_req_word = req_u->req3.tp_feature_req_word;
599}
600
601static void init_prb_bdqc(struct packet_sock *po,
602 struct packet_ring_buffer *rb,
603 struct pgv *pg_vec,
e8e85cc5 604 union tpacket_req_u *req_u)
f6fb8f10 605{
22781a5b 606 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 607 struct tpacket_block_desc *pbd;
f6fb8f10 608
609 memset(p1, 0x0, sizeof(*p1));
610
611 p1->knxt_seq_num = 1;
612 p1->pkbdq = pg_vec;
bc59ba39 613 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 614 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 615 p1->kblk_size = req_u->req3.tp_block_size;
616 p1->knum_blocks = req_u->req3.tp_block_nr;
617 p1->hdrlen = po->tp_hdrlen;
618 p1->version = po->tp_version;
619 p1->last_kactive_blk_num = 0;
ee80fbf3 620 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 621 if (req_u->req3.tp_retire_blk_tov)
622 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
623 else
624 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
625 req_u->req3.tp_block_size);
626 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
627 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
628
dc808110 629 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 630 prb_init_ft_ops(p1, req_u);
e8e85cc5 631 prb_setup_retire_blk_timer(po);
f6fb8f10 632 prb_open_block(p1, pbd);
633}
634
635/* Do NOT update the last_blk_num first.
636 * Assumes sk_buff_head lock is held.
637 */
bc59ba39 638static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 639{
640 mod_timer(&pkc->retire_blk_timer,
641 jiffies + pkc->tov_in_jiffies);
642 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
643}
644
645/*
646 * Timer logic:
647 * 1) We refresh the timer only when we open a block.
648 * By doing this we don't waste cycles refreshing the timer
649 * on packet-by-packet basis.
650 *
651 * With a 1MB block-size, on a 1Gbps line, it will take
652 * i) ~8 ms to fill a block + ii) memcpy etc.
653 * In this cut we are not accounting for the memcpy time.
654 *
655 * So, if the user sets the 'tmo' to 10ms then the timer
656 * will never fire while the block is still getting filled
657 * (which is what we want). However, the user could choose
658 * to close a block early and that's fine.
659 *
660 * But when the timer does fire, we check whether or not to refresh it.
661 * Since the tmo granularity is in msecs, it is not too expensive
662 * to refresh the timer, lets say every '8' msecs.
663 * Either the user can set the 'tmo' or we can derive it based on
664 * a) line-speed and b) block-size.
665 * prb_calc_retire_blk_tmo() calculates the tmo.
666 *
667 */
668static void prb_retire_rx_blk_timer_expired(unsigned long data)
669{
670 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 671 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 672 unsigned int frozen;
bc59ba39 673 struct tpacket_block_desc *pbd;
f6fb8f10 674
675 spin_lock(&po->sk.sk_receive_queue.lock);
676
677 frozen = prb_queue_frozen(pkc);
678 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
679
680 if (unlikely(pkc->delete_blk_timer))
681 goto out;
682
683 /* We only need to plug the race when the block is partially filled.
684 * tpacket_rcv:
685 * lock(); increment BLOCK_NUM_PKTS; unlock()
686 * copy_bits() is in progress ...
687 * timer fires on other cpu:
688 * we can't retire the current block because copy_bits
689 * is in progress.
690 *
691 */
692 if (BLOCK_NUM_PKTS(pbd)) {
693 while (atomic_read(&pkc->blk_fill_in_prog)) {
694 /* Waiting for skb_copy_bits to finish... */
695 cpu_relax();
696 }
697 }
698
699 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
700 if (!frozen) {
41a50d62
AD		 701 if (!BLOCK_NUM_PKTS(pbd)) {
702 /* An empty block. Just refresh the timer. */
703 goto refresh_timer;
704 }
f6fb8f10 705 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
706 if (!prb_dispatch_next_block(pkc, po))
707 goto refresh_timer;
708 else
709 goto out;
710 } else {
711 /* Case 1. Queue was frozen because user-space was
712 * lagging behind.
713 */
714 if (prb_curr_blk_in_use(pkc, pbd)) {
715 /*
716 * Ok, user-space is still behind.
717 * So just refresh the timer.
718 */
719 goto refresh_timer;
720 } else {
721 /* Case 2. queue was frozen,user-space caught up,
722 * now the link went idle && the timer fired.
723 * We don't have a block to close.So we open this
724 * block and restart the timer.
725 * opening a block thaws the queue,restarts timer
726 * Thawing/timer-refresh is a side effect.
727 */
728 prb_open_block(pkc, pbd);
729 goto out;
730 }
731 }
732 }
733
734refresh_timer:
735 _prb_refresh_rx_retire_blk_timer(pkc);
736
737out:
738 spin_unlock(&po->sk.sk_receive_queue.lock);
739}
740
eea49cc9 741static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 742 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 743{
744 /* Flush everything minus the block header */
745
746#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
747 u8 *start, *end;
748
749 start = (u8 *)pbd1;
750
751 /* Skip the block header(we know header WILL fit in 4K) */
752 start += PAGE_SIZE;
753
754 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
755 for (; start < end; start += PAGE_SIZE)
756 flush_dcache_page(pgv_to_page(start));
757
758 smp_wmb();
759#endif
760
761 /* Now update the block status. */
762
763 BLOCK_STATUS(pbd1) = status;
764
765 /* Flush the block header */
766
767#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
768 start = (u8 *)pbd1;
769 flush_dcache_page(pgv_to_page(start));
770
771 smp_wmb();
772#endif
773}
774
775/*
776 * Side effect:
777 *
778 * 1) flush the block
779 * 2) Increment active_blk_num
780 *
781 * Note:We DONT refresh the timer on purpose.
782 * Because almost always the next block will be opened.
783 */
bc59ba39 784static void prb_close_block(struct tpacket_kbdq_core *pkc1,
785 struct tpacket_block_desc *pbd1,
f6fb8f10 786 struct packet_sock *po, unsigned int stat)
787{
788 __u32 status = TP_STATUS_USER | stat;
789
790 struct tpacket3_hdr *last_pkt;
bc59ba39 791 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
da413eec 792 struct sock *sk = &po->sk;
f6fb8f10 793
ee80fbf3 794 if (po->stats.stats3.tp_drops)
f6fb8f10 795 status |= TP_STATUS_LOSING;
796
797 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
798 last_pkt->tp_next_offset = 0;
799
800 /* Get the ts of the last pkt */
801 if (BLOCK_NUM_PKTS(pbd1)) {
802 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
803 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
804 } else {
41a50d62
AD		 805 /* Ok, we tmo'd - so get the current time.
806 *
807 * It shouldn't really happen as we don't close empty
808 * blocks. See prb_retire_rx_blk_timer_expired().
809 */
f6fb8f10 810 struct timespec ts;
811 getnstimeofday(&ts);
812 h1->ts_last_pkt.ts_sec = ts.tv_sec;
813 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
814 }
815
816 smp_wmb();
817
818 /* Flush the block */
819 prb_flush_block(pkc1, pbd1, status);
820
da413eec
DC		 821 sk->sk_data_ready(sk);
822
f6fb8f10 823 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
824}
825
eea49cc9 826static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 827{
828 pkc->reset_pending_on_curr_blk = 0;
829}
830
831/*
832 * Side effect of opening a block:
833 *
834 * 1) prb_queue is thawed.
835 * 2) retire_blk_timer is refreshed.
836 *
837 */
bc59ba39 838static void prb_open_block(struct tpacket_kbdq_core *pkc1,
839 struct tpacket_block_desc *pbd1)
f6fb8f10 840{
841 struct timespec ts;
bc59ba39 842 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 843
844 smp_rmb();
845
8da3056c
DB		 846 /* We could have just memset this but we will lose the
847 * flexibility of making the priv area sticky
848 */
f6fb8f10 849
8da3056c
DB		 850 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
851 BLOCK_NUM_PKTS(pbd1) = 0;
852 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 853
8da3056c
DB		 854 getnstimeofday(&ts);
855
856 h1->ts_first_pkt.ts_sec = ts.tv_sec;
857 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 858
8da3056c
DB		 859 pkc1->pkblk_start = (char *)pbd1;
860 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
861
862 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
863 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
864
865 pbd1->version = pkc1->version;
866 pkc1->prev = pkc1->nxt_offset;
867 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
868
869 prb_thaw_queue(pkc1);
870 _prb_refresh_rx_retire_blk_timer(pkc1);
871
872 smp_wmb();
f6fb8f10 873}
874
875/*
876 * Queue freeze logic:
877 * 1) Assume tp_block_nr = 8 blocks.
878 * 2) At time 't0', user opens Rx ring.
879 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
880 * 4) user-space is either sleeping or processing block '0'.
881 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
882 * it will close block-7,loop around and try to fill block '0'.
883 * call-flow:
884 * __packet_lookup_frame_in_block
885 * prb_retire_current_block()
886 * prb_dispatch_next_block()
887 * |->(BLOCK_STATUS == USER) evaluates to true
888 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
889 * 6) Now there are two cases:
890 * 6.1) Link goes idle right after the queue is frozen.
891 * But remember, the last open_block() refreshed the timer.
892 * When this timer expires,it will refresh itself so that we can
893 * re-open block-0 in near future.
894 * 6.2) Link is busy and keeps on receiving packets. This is a simple
895 * case and __packet_lookup_frame_in_block will check if block-0
896 * is free and can now be re-used.
897 */
eea49cc9 898static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 899 struct packet_sock *po)
900{
901 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 902 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 903}
904
905#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
906
907/*
908 * If the next block is free then we will dispatch it
909 * and return a good offset.
910 * Else, we will freeze the queue.
911 * So, caller must check the return value.
912 */
bc59ba39 913static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 914 struct packet_sock *po)
915{
bc59ba39 916 struct tpacket_block_desc *pbd;
f6fb8f10 917
918 smp_rmb();
919
920 /* 1. Get current block num */
921 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
922
923 /* 2. If this block is currently in_use then freeze the queue */
924 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
925 prb_freeze_queue(pkc, po);
926 return NULL;
927 }
928
929 /*
930 * 3.
931 * open this block and return the offset where the first packet
932 * needs to get stored.
933 */
934 prb_open_block(pkc, pbd);
935 return (void *)pkc->nxt_offset;
936}
937
bc59ba39 938static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 939 struct packet_sock *po, unsigned int status)
940{
bc59ba39 941 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 942
943 /* retire/close the current block */
944 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
945 /*
946 * Plug the case where copy_bits() is in progress on
947 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
948 * have space to copy the pkt in the current block and
949 * called prb_retire_current_block()
950 *
951 * We don't need to worry about the TMO case because
952 * the timer-handler already handled this case.
953 */
954 if (!(status & TP_STATUS_BLK_TMO)) {
955 while (atomic_read(&pkc->blk_fill_in_prog)) {
956 /* Waiting for skb_copy_bits to finish... */
957 cpu_relax();
958 }
959 }
960 prb_close_block(pkc, pbd, po, status);
961 return;
962 }
f6fb8f10 963}
964
eea49cc9 965static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 966 struct tpacket_block_desc *pbd)
f6fb8f10 967{
968 return TP_STATUS_USER & BLOCK_STATUS(pbd);
969}
970
eea49cc9 971static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 972{
973 return pkc->reset_pending_on_curr_blk;
974}
975
eea49cc9 976static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 977{
bc59ba39 978 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 979 atomic_dec(&pkc->blk_fill_in_prog);
980}
981
eea49cc9 982static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 983 struct tpacket3_hdr *ppd)
984{
3958afa1 985 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 986}
987
eea49cc9 988static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 989 struct tpacket3_hdr *ppd)
990{
991 ppd->hv1.tp_rxhash = 0;
992}
993
eea49cc9 994static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 995 struct tpacket3_hdr *ppd)
996{
df8a39de
JP		 997 if (skb_vlan_tag_present(pkc->skb)) {
998 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
a0cdfcf3
AW		 999 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1000 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 1001 } else {
9e67030a 1002 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1003 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1004 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1005 }
1006}
1007
bc59ba39 1008static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1009 struct tpacket3_hdr *ppd)
1010{
a0cdfcf3 1011 ppd->hv1.tp_padding = 0;
f6fb8f10 1012 prb_fill_vlan_info(pkc, ppd);
1013
1014 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1015 prb_fill_rxhash(pkc, ppd);
1016 else
1017 prb_clear_rxhash(pkc, ppd);
1018}
1019
eea49cc9 1020static void prb_fill_curr_block(char *curr,
bc59ba39 1021 struct tpacket_kbdq_core *pkc,
1022 struct tpacket_block_desc *pbd,
f6fb8f10 1023 unsigned int len)
1024{
1025 struct tpacket3_hdr *ppd;
1026
1027 ppd = (struct tpacket3_hdr *)curr;
1028 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1029 pkc->prev = curr;
1030 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1031 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1032 BLOCK_NUM_PKTS(pbd) += 1;
1033 atomic_inc(&pkc->blk_fill_in_prog);
1034 prb_run_all_ft_ops(pkc, ppd);
1035}
1036
1037/* Assumes caller has the sk->rx_queue.lock */
1038static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1039 struct sk_buff *skb,
1040 int status,
1041 unsigned int len
1042 )
1043{
bc59ba39 1044 struct tpacket_kbdq_core *pkc;
1045 struct tpacket_block_desc *pbd;
f6fb8f10 1046 char *curr, *end;
1047
e3192690 1048 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1049 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1050
1051 /* Queue is frozen when user space is lagging behind */
1052 if (prb_queue_frozen(pkc)) {
1053 /*
1054 * Check if that last block which caused the queue to freeze,
1055 * is still in_use by user-space.
1056 */
1057 if (prb_curr_blk_in_use(pkc, pbd)) {
1058 /* Can't record this packet */
1059 return NULL;
1060 } else {
1061 /*
1062 * Ok, the block was released by user-space.
1063 * Now let's open that block.
1064 * opening a block also thaws the queue.
1065 * Thawing is a side effect.
1066 */
1067 prb_open_block(pkc, pbd);
1068 }
1069 }
1070
1071 smp_mb();
1072 curr = pkc->nxt_offset;
1073 pkc->skb = skb;
e3192690 1074 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1075
1076 /* first try the current block */
1077 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1078 prb_fill_curr_block(curr, pkc, pbd, len);
1079 return (void *)curr;
1080 }
1081
1082 /* Ok, close the current block */
1083 prb_retire_current_block(pkc, po, 0);
1084
1085 /* Now, try to dispatch the next block */
1086 curr = (char *)prb_dispatch_next_block(pkc, po);
1087 if (curr) {
1088 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1089 prb_fill_curr_block(curr, pkc, pbd, len);
1090 return (void *)curr;
1091 }
1092
1093 /*
1094 * No free blocks are available.user_space hasn't caught up yet.
1095 * Queue was just frozen and now this packet will get dropped.
1096 */
1097 return NULL;
1098}
1099
eea49cc9 1100static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1101 struct sk_buff *skb,
1102 int status, unsigned int len)
1103{
1104 char *curr = NULL;
1105 switch (po->tp_version) {
1106 case TPACKET_V1:
1107 case TPACKET_V2:
1108 curr = packet_lookup_frame(po, &po->rx_ring,
1109 po->rx_ring.head, status);
1110 return curr;
1111 case TPACKET_V3:
1112 return __packet_lookup_frame_in_block(po, skb, status, len);
1113 default:
1114 WARN(1, "TPACKET version not supported\n");
1115 BUG();
99aa3473 1116 return NULL;
f6fb8f10 1117 }
1118}
1119
eea49cc9 1120static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1121 struct packet_ring_buffer *rb,
77f65ebd 1122 unsigned int idx,
f6fb8f10 1123 int status)
1124{
bc59ba39 1125 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1126 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1127
1128 if (status != BLOCK_STATUS(pbd))
1129 return NULL;
1130 return pbd;
1131}
1132
eea49cc9 1133static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1134{
1135 unsigned int prev;
1136 if (rb->prb_bdqc.kactive_blk_num)
1137 prev = rb->prb_bdqc.kactive_blk_num-1;
1138 else
1139 prev = rb->prb_bdqc.knum_blocks-1;
1140 return prev;
1141}
1142
1143/* Assumes caller has held the rx_queue.lock */
eea49cc9 1144static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1145 struct packet_ring_buffer *rb,
1146 int status)
1147{
1148 unsigned int previous = prb_previous_blk_num(rb);
1149 return prb_lookup_block(po, rb, previous, status);
1150}
1151
eea49cc9 1152static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1153 struct packet_ring_buffer *rb,
1154 int status)
1155{
1156 if (po->tp_version <= TPACKET_V2)
1157 return packet_previous_frame(po, rb, status);
1158
1159 return __prb_previous_block(po, rb, status);
1160}
1161
eea49cc9 1162static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1163 struct packet_ring_buffer *rb)
1164{
1165 switch (po->tp_version) {
1166 case TPACKET_V1:
1167 case TPACKET_V2:
1168 return packet_increment_head(rb);
1169 case TPACKET_V3:
1170 default:
1171 WARN(1, "TPACKET version not supported.\n");
1172 BUG();
1173 return;
1174 }
1175}
1176
eea49cc9 1177static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1178 struct packet_ring_buffer *rb,
1179 int status)
1180{
1181 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1182 return packet_lookup_frame(po, rb, previous, status);
1183}
1184
eea49cc9 1185static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1186{
1187 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1188}
1189
b0138408
DB		 1190static void packet_inc_pending(struct packet_ring_buffer *rb)
1191{
1192 this_cpu_inc(*rb->pending_refcnt);
1193}
1194
1195static void packet_dec_pending(struct packet_ring_buffer *rb)
1196{
1197 this_cpu_dec(*rb->pending_refcnt);
1198}
1199
1200static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1201{
1202 unsigned int refcnt = 0;
1203 int cpu;
1204
1205 /* We don't use pending refcount in rx_ring. */
1206 if (rb->pending_refcnt == NULL)
1207 return 0;
1208
1209 for_each_possible_cpu(cpu)
1210 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1211
1212 return refcnt;
1213}
1214
1215static int packet_alloc_pending(struct packet_sock *po)
1216{
1217 po->rx_ring.pending_refcnt = NULL;
1218
1219 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1220 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1221 return -ENOBUFS;
1222
1223 return 0;
1224}
1225
1226static void packet_free_pending(struct packet_sock *po)
1227{
1228 free_percpu(po->tx_ring.pending_refcnt);
1229}
1230
9954729b
WB		 1231#define ROOM_POW_OFF 2
1232#define ROOM_NONE 0x0
1233#define ROOM_LOW 0x1
1234#define ROOM_NORMAL 0x2
1235
1236static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
77f65ebd 1237{
9954729b
WB		 1238 int idx, len;
1239
1240 len = po->rx_ring.frame_max + 1;
1241 idx = po->rx_ring.head;
1242 if (pow_off)
1243 idx += len >> pow_off;
1244 if (idx >= len)
1245 idx -= len;
1246 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1247}
1248
1249static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1250{
1251 int idx, len;
1252
1253 len = po->rx_ring.prb_bdqc.knum_blocks;
1254 idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1255 if (pow_off)
1256 idx += len >> pow_off;
1257 if (idx >= len)
1258 idx -= len;
1259 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1260}
77f65ebd 1261
2ccdbaa6 1262static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
9954729b
WB		 1263{
1264 struct sock *sk = &po->sk;
1265 int ret = ROOM_NONE;
1266
1267 if (po->prot_hook.func != tpacket_rcv) {
1268 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
2ccdbaa6 1269 - (skb ? skb->truesize : 0);
9954729b
WB		 1270 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1271 return ROOM_NORMAL;
1272 else if (avail > 0)
1273 return ROOM_LOW;
1274 else
1275 return ROOM_NONE;
1276 }
77f65ebd 1277
9954729b
WB		 1278 if (po->tp_version == TPACKET_V3) {
1279 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1280 ret = ROOM_NORMAL;
1281 else if (__tpacket_v3_has_room(po, 0))
1282 ret = ROOM_LOW;
1283 } else {
1284 if (__tpacket_has_room(po, ROOM_POW_OFF))
1285 ret = ROOM_NORMAL;
1286 else if (__tpacket_has_room(po, 0))
1287 ret = ROOM_LOW;
1288 }
2ccdbaa6
WB		 1289
1290 return ret;
1291}
1292
1293static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1294{
1295 int ret;
1296 bool has_room;
1297
54d7c01d
WB		 1298 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1299 ret = __packet_rcv_has_room(po, skb);
2ccdbaa6
WB		 1300 has_room = ret == ROOM_NORMAL;
1301 if (po->pressure == has_room)
54d7c01d
WB		 1302 po->pressure = !has_room;
1303 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
77f65ebd 1304
9954729b 1305 return ret;
77f65ebd
WB		 1306}
1307
1da177e4
LT		 1308static void packet_sock_destruct(struct sock *sk)
1309{
ed85b565
RC		 1310 skb_queue_purge(&sk->sk_error_queue);
1311
547b792c
IJ		 1312 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1313 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1314
1315 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1316 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1317 return;
1318 }
1319
17ab56a2 1320 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1321}
1322
3b3a5b0a
WB		 1323static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1324{
1325 u32 rxhash;
1326 int i, count = 0;
1327
1328 rxhash = skb_get_hash(skb);
1329 for (i = 0; i < ROLLOVER_HLEN; i++)
1330 if (po->rollover->history[i] == rxhash)
1331 count++;
1332
1333 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1334 return count > (ROLLOVER_HLEN >> 1);
1335}
1336
77f65ebd
WB		 1337static unsigned int fanout_demux_hash(struct packet_fanout *f,
1338 struct sk_buff *skb,
1339 unsigned int num)
dc99f600 1340{
61b905da 1341 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1342}
1343
77f65ebd
WB		 1344static unsigned int fanout_demux_lb(struct packet_fanout *f,
1345 struct sk_buff *skb,
1346 unsigned int num)
dc99f600 1347{
468479e6 1348 unsigned int val = atomic_inc_return(&f->rr_cur);
dc99f600 1349
468479e6 1350 return val % num;
77f65ebd
WB		 1351}
1352
1353static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1354 struct sk_buff *skb,
1355 unsigned int num)
1356{
1357 return smp_processor_id() % num;
dc99f600
DM		 1358}
1359
5df0ddfb
DB		 1360static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1361 struct sk_buff *skb,
1362 unsigned int num)
1363{
f337db64 1364 return prandom_u32_max(num);
5df0ddfb
DB		 1365}
1366
77f65ebd
WB		 1367static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1368 struct sk_buff *skb,
ad377cab 1369 unsigned int idx, bool try_self,
77f65ebd 1370 unsigned int num)
95ec3eb4 1371{
4633c9e0 1372 struct packet_sock *po, *po_next, *po_skip = NULL;
a9b63918 1373 unsigned int i, j, room = ROOM_NONE;
95ec3eb4 1374
0648ab70 1375 po = pkt_sk(f->arr[idx]);
3b3a5b0a
WB		 1376
1377 if (try_self) {
1378 room = packet_rcv_has_room(po, skb);
1379 if (room == ROOM_NORMAL ||
1380 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1381 return idx;
4633c9e0 1382 po_skip = po;
3b3a5b0a 1383 }
ad377cab 1384
0648ab70 1385 i = j = min_t(int, po->rollover->sock, num - 1);
77f65ebd 1386 do {
2ccdbaa6 1387 po_next = pkt_sk(f->arr[i]);
4633c9e0 1388 if (po_next != po_skip && !po_next->pressure &&
2ccdbaa6 1389 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
77f65ebd 1390 if (i != j)
0648ab70 1391 po->rollover->sock = i;
a9b63918
WB		 1392 atomic_long_inc(&po->rollover->num);
1393 if (room == ROOM_LOW)
1394 atomic_long_inc(&po->rollover->num_huge);
77f65ebd
WB		 1395 return i;
1396 }
ad377cab 1397
77f65ebd
WB		 1398 if (++i == num)
1399 i = 0;
1400 } while (i != j);
1401
a9b63918 1402 atomic_long_inc(&po->rollover->num_failed);
77f65ebd
WB		 1403 return idx;
1404}
1405
2d36097d
NH		 1406static unsigned int fanout_demux_qm(struct packet_fanout *f,
1407 struct sk_buff *skb,
1408 unsigned int num)
1409{
1410 return skb_get_queue_mapping(skb) % num;
1411}
1412
77f65ebd
WB		 1413static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1414{
1415 return f->flags & (flag >> 8);
95ec3eb4
DM		 1416}
1417
95ec3eb4
DM		 1418static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1419 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1420{
1421 struct packet_fanout *f = pt->af_packet_priv;
f98f4514 1422 unsigned int num = READ_ONCE(f->num_members);
dc99f600 1423 struct packet_sock *po;
77f65ebd 1424 unsigned int idx;
dc99f600
DM		 1425
1426 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1427 !num) {
1428 kfree_skb(skb);
1429 return 0;
1430 }
1431
3f34b24a
AD		 1432 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1433 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1434 if (!skb)
1435 return 0;
1436 }
95ec3eb4
DM		 1437 switch (f->type) {
1438 case PACKET_FANOUT_HASH:
1439 default:
77f65ebd 1440 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1441 break;
1442 case PACKET_FANOUT_LB:
77f65ebd 1443 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1444 break;
1445 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1446 idx = fanout_demux_cpu(f, skb, num);
1447 break;
5df0ddfb
DB		 1448 case PACKET_FANOUT_RND:
1449 idx = fanout_demux_rnd(f, skb, num);
1450 break;
2d36097d
NH		 1451 case PACKET_FANOUT_QM:
1452 idx = fanout_demux_qm(f, skb, num);
1453 break;
77f65ebd 1454 case PACKET_FANOUT_ROLLOVER:
ad377cab 1455 idx = fanout_demux_rollover(f, skb, 0, false, num);
95ec3eb4 1456 break;
dc99f600
DM		 1457 }
1458
ad377cab
WB		 1459 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1460 idx = fanout_demux_rollover(f, skb, idx, true, num);
dc99f600 1461
ad377cab 1462 po = pkt_sk(f->arr[idx]);
dc99f600
DM		 1463 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1464}
1465
fff3321d
PE		 1466DEFINE_MUTEX(fanout_mutex);
1467EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1468static LIST_HEAD(fanout_list);
1469
1470static void __fanout_link(struct sock *sk, struct packet_sock *po)
1471{
1472 struct packet_fanout *f = po->fanout;
1473
1474 spin_lock(&f->lock);
1475 f->arr[f->num_members] = sk;
1476 smp_wmb();
1477 f->num_members++;
1478 spin_unlock(&f->lock);
1479}
1480
1481static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1482{
1483 struct packet_fanout *f = po->fanout;
1484 int i;
1485
1486 spin_lock(&f->lock);
1487 for (i = 0; i < f->num_members; i++) {
1488 if (f->arr[i] == sk)
1489 break;
1490 }
1491 BUG_ON(i >= f->num_members);
1492 f->arr[i] = f->arr[f->num_members - 1];
1493 f->num_members--;
1494 spin_unlock(&f->lock);
1495}
1496
d4dd8aee 1497static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1498{
d4dd8aee 1499 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1500 return true;
1501
1502 return false;
1503}
1504
7736d33f 1505static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1506{
1507 struct packet_sock *po = pkt_sk(sk);
1508 struct packet_fanout *f, *match;
7736d33f 1509 u8 type = type_flags & 0xff;
77f65ebd 1510 u8 flags = type_flags >> 8;
dc99f600
DM		 1511 int err;
1512
1513 switch (type) {
77f65ebd
WB		 1514 case PACKET_FANOUT_ROLLOVER:
1515 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1516 return -EINVAL;
dc99f600
DM		 1517 case PACKET_FANOUT_HASH:
1518 case PACKET_FANOUT_LB:
95ec3eb4 1519 case PACKET_FANOUT_CPU:
5df0ddfb 1520 case PACKET_FANOUT_RND:
2d36097d 1521 case PACKET_FANOUT_QM:
dc99f600
DM		 1522 break;
1523 default:
1524 return -EINVAL;
1525 }
1526
1527 if (!po->running)
1528 return -EINVAL;
1529
1530 if (po->fanout)
1531 return -EALREADY;
1532
4633c9e0
WB		 1533 if (type == PACKET_FANOUT_ROLLOVER ||
1534 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
0648ab70
WB		 1535 po->rollover = kzalloc(sizeof(*po->rollover), GFP_KERNEL);
1536 if (!po->rollover)
1537 return -ENOMEM;
a9b63918
WB		 1538 atomic_long_set(&po->rollover->num, 0);
1539 atomic_long_set(&po->rollover->num_huge, 0);
1540 atomic_long_set(&po->rollover->num_failed, 0);
0648ab70
WB		 1541 }
1542
dc99f600
DM		 1543 mutex_lock(&fanout_mutex);
1544 match = NULL;
1545 list_for_each_entry(f, &fanout_list, list) {
1546 if (f->id == id &&
1547 read_pnet(&f->net) == sock_net(sk)) {
1548 match = f;
1549 break;
1550 }
1551 }
afe62c68 1552 err = -EINVAL;
77f65ebd 1553 if (match && match->flags != flags)
afe62c68 1554 goto out;
dc99f600 1555 if (!match) {
afe62c68 1556 err = -ENOMEM;
dc99f600 1557 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1558 if (!match)
1559 goto out;
1560 write_pnet(&match->net, sock_net(sk));
1561 match->id = id;
1562 match->type = type;
77f65ebd 1563 match->flags = flags;
afe62c68
ED		 1564 atomic_set(&match->rr_cur, 0);
1565 INIT_LIST_HEAD(&match->list);
1566 spin_lock_init(&match->lock);
1567 atomic_set(&match->sk_ref, 0);
1568 match->prot_hook.type = po->prot_hook.type;
1569 match->prot_hook.dev = po->prot_hook.dev;
1570 match->prot_hook.func = packet_rcv_fanout;
1571 match->prot_hook.af_packet_priv = match;
c0de08d0 1572 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1573 dev_add_pack(&match->prot_hook);
1574 list_add(&match->list, &fanout_list);
dc99f600 1575 }
afe62c68
ED		 1576 err = -EINVAL;
1577 if (match->type == type &&
1578 match->prot_hook.type == po->prot_hook.type &&
1579 match->prot_hook.dev == po->prot_hook.dev) {
1580 err = -ENOSPC;
1581 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1582 __dev_remove_pack(&po->prot_hook);
1583 po->fanout = match;
1584 atomic_inc(&match->sk_ref);
1585 __fanout_link(sk, po);
1586 err = 0;
dc99f600
DM		 1587 }
1588 }
afe62c68 1589out:
dc99f600 1590 mutex_unlock(&fanout_mutex);
0648ab70
WB		 1591 if (err) {
1592 kfree(po->rollover);
1593 po->rollover = NULL;
1594 }
dc99f600
DM		 1595 return err;
1596}
1597
1598static void fanout_release(struct sock *sk)
1599{
1600 struct packet_sock *po = pkt_sk(sk);
1601 struct packet_fanout *f;
1602
1603 f = po->fanout;
1604 if (!f)
1605 return;
1606
fff3321d 1607 mutex_lock(&fanout_mutex);
dc99f600
DM		 1608 po->fanout = NULL;
1609
dc99f600
DM		 1610 if (atomic_dec_and_test(&f->sk_ref)) {
1611 list_del(&f->list);
1612 dev_remove_pack(&f->prot_hook);
1613 kfree(f);
1614 }
1615 mutex_unlock(&fanout_mutex);
0648ab70 1616
59f21118
WB		 1617 if (po->rollover)
1618 kfree_rcu(po->rollover, rcu);
dc99f600 1619}
1da177e4 1620
90ddc4f0 1621static const struct proto_ops packet_ops;
1da177e4 1622
90ddc4f0 1623static const struct proto_ops packet_ops_spkt;
1da177e4 1624
40d4e3df
ED		 1625static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1626 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1627{
1628 struct sock *sk;
1629 struct sockaddr_pkt *spkt;
1630
1631 /*
1632 * When we registered the protocol we saved the socket in the data
1633 * field for just this event.
1634 */
1635
1636 sk = pt->af_packet_priv;
1ce4f28b 1637
1da177e4
LT		 1638 /*
1639 * Yank back the headers [hope the device set this
1640 * right or kerboom...]
1641 *
1642 * Incoming packets have ll header pulled,
1643 * push it back.
1644 *
98e399f8 1645 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1646 * so that this procedure is noop.
1647 */
1648
1649 if (skb->pkt_type == PACKET_LOOPBACK)
1650 goto out;
1651
09ad9bc7 1652 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1653 goto out;
1654
40d4e3df
ED		 1655 skb = skb_share_check(skb, GFP_ATOMIC);
1656 if (skb == NULL)
1da177e4
LT		 1657 goto oom;
1658
1659 /* drop any routing info */
adf30907 1660 skb_dst_drop(skb);
1da177e4 1661
84531c24
PO		 1662 /* drop conntrack reference */
1663 nf_reset(skb);
1664
ffbc6111 1665 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1666
98e399f8 1667 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1668
1669 /*
1670 * The SOCK_PACKET socket receives _all_ frames.
1671 */
1672
1673 spkt->spkt_family = dev->type;
1674 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1675 spkt->spkt_protocol = skb->protocol;
1676
1677 /*
1678 * Charge the memory to the socket. This is done specifically
1679 * to prevent sockets using all the memory up.
1680 */
1681
40d4e3df 1682 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1683 return 0;
1684
1685out:
1686 kfree_skb(skb);
1687oom:
1688 return 0;
1689}
1690
1691
1692/*
1693 * Output a raw packet to a device layer. This bypasses all the other
1694 * protocol layers and you must therefore supply it with a complete frame
1695 */
1ce4f28b 1696
1b784140
YX		 1697static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1698 size_t len)
1da177e4
LT		 1699{
1700 struct sock *sk = sock->sk;
342dfc30 1701 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1702 struct sk_buff *skb = NULL;
1da177e4 1703 struct net_device *dev;
40d4e3df 1704 __be16 proto = 0;
1da177e4 1705 int err;
3bdc0eba 1706 int extra_len = 0;
1ce4f28b 1707
1da177e4 1708 /*
1ce4f28b 1709 * Get and verify the address.
1da177e4
LT		 1710 */
1711
40d4e3df 1712 if (saddr) {
1da177e4 1713 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1714 return -EINVAL;
1715 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1716 proto = saddr->spkt_protocol;
1717 } else
1718 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1719
1720 /*
1ce4f28b 1721 * Find the device first to size check it
1da177e4
LT		 1722 */
1723
de74e92a 1724 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1725retry:
654d1f8a
ED		 1726 rcu_read_lock();
1727 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1728 err = -ENODEV;
1729 if (dev == NULL)
1730 goto out_unlock;
1ce4f28b 1731
d5e76b0a
DM		 1732 err = -ENETDOWN;
1733 if (!(dev->flags & IFF_UP))
1734 goto out_unlock;
1735
1da177e4 1736 /*
40d4e3df
ED		 1737 * You may not queue a frame bigger than the mtu. This is the lowest level
1738 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1739 */
1ce4f28b 1740
3bdc0eba
BG		 1741 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1742 if (!netif_supports_nofcs(dev)) {
1743 err = -EPROTONOSUPPORT;
1744 goto out_unlock;
1745 }
1746 extra_len = 4; /* We're doing our own CRC */
1747 }
1748
1da177e4 1749 err = -EMSGSIZE;
3bdc0eba 1750 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1751 goto out_unlock;
1752
1a35ca80
ED		 1753 if (!skb) {
1754 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1755 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1756 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1757
1758 rcu_read_unlock();
4ce40912 1759 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1760 if (skb == NULL)
1761 return -ENOBUFS;
1762 /* FIXME: Save some space for broken drivers that write a hard
1763 * header at transmission time by themselves. PPP is the notable
1764 * one here. This should really be fixed at the driver level.
1765 */
1766 skb_reserve(skb, reserved);
1767 skb_reset_network_header(skb);
1768
1769 /* Try to align data part correctly */
1770 if (hhlen) {
1771 skb->data -= hhlen;
1772 skb->tail -= hhlen;
1773 if (len < hhlen)
1774 skb_reset_network_header(skb);
1775 }
6ce8e9ce 1776 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1a35ca80
ED		 1777 if (err)
1778 goto out_free;
1779 goto retry;
1da177e4
LT		 1780 }
1781
3bdc0eba 1782 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1783 /* Earlier code assumed this would be a VLAN pkt,
1784 * double-check this now that we have the actual
1785 * packet in hand.
1786 */
1787 struct ethhdr *ehdr;
1788 skb_reset_mac_header(skb);
1789 ehdr = eth_hdr(skb);
1790 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1791 err = -EMSGSIZE;
1792 goto out_unlock;
1793 }
1794 }
1a35ca80 1795
1da177e4
LT		 1796 skb->protocol = proto;
1797 skb->dev = dev;
1798 skb->priority = sk->sk_priority;
2d37a186 1799 skb->mark = sk->sk_mark;
bf84a010
DB		 1800
1801 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1802
3bdc0eba
BG		 1803 if (unlikely(extra_len == 4))
1804 skb->no_fcs = 1;
1805
40893fd0 1806 skb_probe_transport_header(skb, 0);
c1aad275 1807
1da177e4 1808 dev_queue_xmit(skb);
654d1f8a 1809 rcu_read_unlock();
40d4e3df 1810 return len;
1da177e4 1811
1da177e4 1812out_unlock:
654d1f8a 1813 rcu_read_unlock();
1a35ca80
ED		 1814out_free:
1815 kfree_skb(skb);
1da177e4
LT		 1816 return err;
1817}
1da177e4 1818
eea49cc9 1819static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1820 const struct sock *sk,
dbcb5855 1821 unsigned int res)
1da177e4
LT		 1822{
1823 struct sk_filter *filter;
fda9ef5d 1824
80f8f102
ED		 1825 rcu_read_lock();
1826 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1827 if (filter != NULL)
0a14842f 1828 res = SK_RUN_FILTER(filter, skb);
80f8f102 1829 rcu_read_unlock();
1da177e4 1830
dbcb5855 1831 return res;
1da177e4
LT		 1832}
1833
1834/*
62ab0812
ED		 1835 * This function makes lazy skb cloning in hope that most of packets
1836 * are discarded by BPF.
1837 *
1838 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1839 * and skb->cb are mangled. It works because (and until) packets
1840 * falling here are owned by current CPU. Output packets are cloned
1841 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1842 * sequencially, so that if we return skb to original state on exit,
1843 * we will not harm anyone.
1da177e4
LT		 1844 */
1845
40d4e3df
ED		 1846static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1847 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1848{
1849 struct sock *sk;
1850 struct sockaddr_ll *sll;
1851 struct packet_sock *po;
40d4e3df 1852 u8 *skb_head = skb->data;
1da177e4 1853 int skb_len = skb->len;
dbcb5855 1854 unsigned int snaplen, res;
1da177e4
LT		 1855
1856 if (skb->pkt_type == PACKET_LOOPBACK)
1857 goto drop;
1858
1859 sk = pt->af_packet_priv;
1860 po = pkt_sk(sk);
1861
09ad9bc7 1862 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1863 goto drop;
1864
1da177e4
LT		 1865 skb->dev = dev;
1866
3b04ddde 1867 if (dev->header_ops) {
1da177e4 1868 /* The device has an explicit notion of ll header,
62ab0812
ED		 1869 * exported to higher levels.
1870 *
1871 * Otherwise, the device hides details of its frame
1872 * structure, so that corresponding packet head is
1873 * never delivered to user.
1da177e4
LT		 1874 */
1875 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1876 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1877 else if (skb->pkt_type == PACKET_OUTGOING) {
1878 /* Special case: outgoing packets have ll header at head */
bbe735e4 1879 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1880 }
1881 }
1882
1883 snaplen = skb->len;
1884
dbcb5855
DM		 1885 res = run_filter(skb, sk, snaplen);
1886 if (!res)
fda9ef5d 1887 goto drop_n_restore;
dbcb5855
DM		 1888 if (snaplen > res)
1889 snaplen = res;
1da177e4 1890
0fd7bac6 1891 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1892 goto drop_n_acct;
1893
1894 if (skb_shared(skb)) {
1895 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1896 if (nskb == NULL)
1897 goto drop_n_acct;
1898
1899 if (skb_head != skb->data) {
1900 skb->data = skb_head;
1901 skb->len = skb_len;
1902 }
abc4e4fa 1903 consume_skb(skb);
1da177e4
LT		 1904 skb = nskb;
1905 }
1906
b4772ef8 1907 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
ffbc6111
HX		 1908
1909 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4 1910 sll->sll_hatype = dev->type;
1da177e4 1911 sll->sll_pkttype = skb->pkt_type;
8032b464 1912 if (unlikely(po->origdev))
80feaacb
PWJ		 1913 sll->sll_ifindex = orig_dev->ifindex;
1914 else
1915 sll->sll_ifindex = dev->ifindex;
1da177e4 1916
b95cce35 1917 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1918
2472d761
EB		 1919 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
1920 * Use their space for storing the original skb length.
1921 */
1922 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
8dc41944 1923
1da177e4
LT		 1924 if (pskb_trim(skb, snaplen))
1925 goto drop_n_acct;
1926
1927 skb_set_owner_r(skb, sk);
1928 skb->dev = NULL;
adf30907 1929 skb_dst_drop(skb);
1da177e4 1930
84531c24
PO		 1931 /* drop conntrack reference */
1932 nf_reset(skb);
1933
1da177e4 1934 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1935 po->stats.stats1.tp_packets++;
3bc3b96f 1936 sock_skb_set_dropcount(sk, skb);
1da177e4
LT		 1937 __skb_queue_tail(&sk->sk_receive_queue, skb);
1938 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 1939 sk->sk_data_ready(sk);
1da177e4
LT		 1940 return 0;
1941
1942drop_n_acct:
7091fbd8 1943 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1944 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1945 atomic_inc(&sk->sk_drops);
1946 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1947
1948drop_n_restore:
1949 if (skb_head != skb->data && skb_shared(skb)) {
1950 skb->data = skb_head;
1951 skb->len = skb_len;
1952 }
1953drop:
ead2ceb0 1954 consume_skb(skb);
1da177e4
LT		 1955 return 0;
1956}
1957
40d4e3df
ED		 1958static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1959 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1960{
1961 struct sock *sk;
1962 struct packet_sock *po;
1963 struct sockaddr_ll *sll;
184f489e 1964 union tpacket_uhdr h;
40d4e3df 1965 u8 *skb_head = skb->data;
1da177e4 1966 int skb_len = skb->len;
dbcb5855 1967 unsigned int snaplen, res;
f6fb8f10 1968 unsigned long status = TP_STATUS_USER;
bbd6ef87 1969 unsigned short macoff, netoff, hdrlen;
1da177e4 1970 struct sk_buff *copy_skb = NULL;
bbd6ef87 1971 struct timespec ts;
b9c32fb2 1972 __u32 ts_status;
1da177e4 1973
51846355
AW		 1974 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1975 * We may add members to them until current aligned size without forcing
1976 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1977 */
1978 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1979 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1980
1da177e4
LT		 1981 if (skb->pkt_type == PACKET_LOOPBACK)
1982 goto drop;
1983
1984 sk = pt->af_packet_priv;
1985 po = pkt_sk(sk);
1986
09ad9bc7 1987 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1988 goto drop;
1989
3b04ddde 1990 if (dev->header_ops) {
1da177e4 1991 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1992 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1993 else if (skb->pkt_type == PACKET_OUTGOING) {
1994 /* Special case: outgoing packets have ll header at head */
bbe735e4 1995 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1996 }
1997 }
1998
1999 snaplen = skb->len;
2000
dbcb5855
DM		 2001 res = run_filter(skb, sk, snaplen);
2002 if (!res)
fda9ef5d 2003 goto drop_n_restore;
68c2e5de
AD		 2004
2005 if (skb->ip_summed == CHECKSUM_PARTIAL)
2006 status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 2007 else if (skb->pkt_type != PACKET_OUTGOING &&
2008 (skb->ip_summed == CHECKSUM_COMPLETE ||
2009 skb_csum_unnecessary(skb)))
2010 status |= TP_STATUS_CSUM_VALID;
68c2e5de 2011
dbcb5855
DM		 2012 if (snaplen > res)
2013 snaplen = res;
1da177e4
LT		 2014
2015 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 2016 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2017 po->tp_reserve;
1da177e4 2018 } else {
95c96174 2019 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 2020 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 2021 (maclen < 16 ? 16 : maclen)) +
2022 po->tp_reserve;
1da177e4
LT		 2023 macoff = netoff - maclen;
2024 }
f6fb8f10 2025 if (po->tp_version <= TPACKET_V2) {
2026 if (macoff + snaplen > po->rx_ring.frame_size) {
2027 if (po->copy_thresh &&
0fd7bac6 2028 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 2029 if (skb_shared(skb)) {
2030 copy_skb = skb_clone(skb, GFP_ATOMIC);
2031 } else {
2032 copy_skb = skb_get(skb);
2033 skb_head = skb->data;
2034 }
2035 if (copy_skb)
2036 skb_set_owner_r(copy_skb, sk);
1da177e4 2037 }
f6fb8f10 2038 snaplen = po->rx_ring.frame_size - macoff;
2039 if ((int)snaplen < 0)
2040 snaplen = 0;
1da177e4 2041 }
dc808110
ED		 2042 } else if (unlikely(macoff + snaplen >
2043 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2044 u32 nval;
2045
2046 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2047 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2048 snaplen, nval, macoff);
2049 snaplen = nval;
2050 if (unlikely((int)snaplen < 0)) {
2051 snaplen = 0;
2052 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2053 }
1da177e4 2054 }
1da177e4 2055 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 2056 h.raw = packet_current_rx_frame(po, skb,
2057 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 2058 if (!h.raw)
1da177e4 2059 goto ring_is_full;
f6fb8f10 2060 if (po->tp_version <= TPACKET_V2) {
2061 packet_increment_rx_head(po, &po->rx_ring);
2062 /*
2063 * LOSING will be reported till you read the stats,
2064 * because it's COR - Clear On Read.
2065 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2066 * at packet level.
2067 */
ee80fbf3 2068 if (po->stats.stats1.tp_drops)
f6fb8f10 2069 status |= TP_STATUS_LOSING;
2070 }
ee80fbf3 2071 po->stats.stats1.tp_packets++;
1da177e4
LT		 2072 if (copy_skb) {
2073 status |= TP_STATUS_COPY;
2074 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2075 }
1da177e4
LT		 2076 spin_unlock(&sk->sk_receive_queue.lock);
2077
bbd6ef87 2078 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 2079
2080 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 2081 getnstimeofday(&ts);
1da177e4 2082
b9c32fb2
DB		 2083 status |= ts_status;
2084
bbd6ef87
PM		 2085 switch (po->tp_version) {
2086 case TPACKET_V1:
2087 h.h1->tp_len = skb->len;
2088 h.h1->tp_snaplen = snaplen;
2089 h.h1->tp_mac = macoff;
2090 h.h1->tp_net = netoff;
4b457bdf
DB		 2091 h.h1->tp_sec = ts.tv_sec;
2092 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 2093 hdrlen = sizeof(*h.h1);
2094 break;
2095 case TPACKET_V2:
2096 h.h2->tp_len = skb->len;
2097 h.h2->tp_snaplen = snaplen;
2098 h.h2->tp_mac = macoff;
2099 h.h2->tp_net = netoff;
bbd6ef87
PM		 2100 h.h2->tp_sec = ts.tv_sec;
2101 h.h2->tp_nsec = ts.tv_nsec;
df8a39de
JP		 2102 if (skb_vlan_tag_present(skb)) {
2103 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 2104 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2105 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2106 } else {
2107 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2108 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2109 }
e4d26f4b 2110 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2111 hdrlen = sizeof(*h.h2);
2112 break;
f6fb8f10 2113 case TPACKET_V3:
2114 /* tp_nxt_offset,vlan are already populated above.
2115 * So DONT clear those fields here
2116 */
2117 h.h3->tp_status |= status;
2118 h.h3->tp_len = skb->len;
2119 h.h3->tp_snaplen = snaplen;
2120 h.h3->tp_mac = macoff;
2121 h.h3->tp_net = netoff;
f6fb8f10 2122 h.h3->tp_sec = ts.tv_sec;
2123 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2124 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2125 hdrlen = sizeof(*h.h3);
2126 break;
bbd6ef87
PM		 2127 default:
2128 BUG();
2129 }
1da177e4 2130
bbd6ef87 2131 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2132 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2133 sll->sll_family = AF_PACKET;
2134 sll->sll_hatype = dev->type;
2135 sll->sll_protocol = skb->protocol;
2136 sll->sll_pkttype = skb->pkt_type;
8032b464 2137 if (unlikely(po->origdev))
80feaacb
PWJ		 2138 sll->sll_ifindex = orig_dev->ifindex;
2139 else
2140 sll->sll_ifindex = dev->ifindex;
1da177e4 2141
e16aa207 2142 smp_mb();
f0d4eb29 2143
f6dafa95 2144#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2145 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2146 u8 *start, *end;
2147
f0d4eb29
DB		 2148 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2149 macoff + snaplen);
2150
2151 for (start = h.raw; start < end; start += PAGE_SIZE)
2152 flush_dcache_page(pgv_to_page(start));
1da177e4 2153 }
f0d4eb29 2154 smp_wmb();
f6dafa95 2155#endif
f0d4eb29 2156
da413eec 2157 if (po->tp_version <= TPACKET_V2) {
f6fb8f10 2158 __packet_set_status(po, h.raw, status);
da413eec
DC		 2159 sk->sk_data_ready(sk);
2160 } else {
f6fb8f10 2161 prb_clear_blk_fill_status(&po->rx_ring);
da413eec 2162 }
1da177e4
LT		 2163
2164drop_n_restore:
2165 if (skb_head != skb->data && skb_shared(skb)) {
2166 skb->data = skb_head;
2167 skb->len = skb_len;
2168 }
2169drop:
1ce4f28b 2170 kfree_skb(skb);
1da177e4
LT		 2171 return 0;
2172
2173ring_is_full:
ee80fbf3 2174 po->stats.stats1.tp_drops++;
1da177e4
LT		 2175 spin_unlock(&sk->sk_receive_queue.lock);
2176
676d2369 2177 sk->sk_data_ready(sk);
acb5d75b 2178 kfree_skb(copy_skb);
1da177e4
LT		 2179 goto drop_n_restore;
2180}
2181
69e3c75f
JB		 2182static void tpacket_destruct_skb(struct sk_buff *skb)
2183{
2184 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2185
69e3c75f 2186 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2187 void *ph;
b9c32fb2
DB		 2188 __u32 ts;
2189
69e3c75f 2190 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2191 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2192
2193 ts = __packet_set_timestamp(po, ph, skb);
2194 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2195 }
2196
2197 sock_wfree(skb);
2198}
2199
9c707762
WB		 2200static bool ll_header_truncated(const struct net_device *dev, int len)
2201{
2202 /* net device doesn't like empty head */
2203 if (unlikely(len <= dev->hard_header_len)) {
eee2f04b 2204 net_warn_ratelimited("%s: packet size is too short (%d <= %d)\n",
9c707762
WB		 2205 current->comm, len, dev->hard_header_len);
2206 return true;
2207 }
2208
2209 return false;
2210}
2211
40d4e3df
ED		 2212static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2213 void *frame, struct net_device *dev, int size_max,
ae641949 2214 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2215{
184f489e 2216 union tpacket_uhdr ph;
09effa67 2217 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2218 struct socket *sock = po->sk.sk_socket;
2219 struct page *page;
2220 void *data;
2221 int err;
2222
2223 ph.raw = frame;
2224
2225 skb->protocol = proto;
2226 skb->dev = dev;
2227 skb->priority = po->sk.sk_priority;
2d37a186 2228 skb->mark = po->sk.sk_mark;
2e31396f 2229 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2230 skb_shinfo(skb)->destructor_arg = ph.raw;
2231
2232 switch (po->tp_version) {
2233 case TPACKET_V2:
2234 tp_len = ph.h2->tp_len;
2235 break;
2236 default:
2237 tp_len = ph.h1->tp_len;
2238 break;
2239 }
09effa67
DM		 2240 if (unlikely(tp_len > size_max)) {
2241 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2242 return -EMSGSIZE;
2243 }
69e3c75f 2244
ae641949 2245 skb_reserve(skb, hlen);
69e3c75f 2246 skb_reset_network_header(skb);
c1aad275 2247
d346a3fa
DB		 2248 if (!packet_use_direct_xmit(po))
2249 skb_probe_transport_header(skb, 0);
2250 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2251 int off_min, off_max, off;
2252 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2253 off_max = po->tx_ring.frame_size - tp_len;
2254 if (sock->type == SOCK_DGRAM) {
2255 switch (po->tp_version) {
2256 case TPACKET_V2:
2257 off = ph.h2->tp_net;
2258 break;
2259 default:
2260 off = ph.h1->tp_net;
2261 break;
2262 }
2263 } else {
2264 switch (po->tp_version) {
2265 case TPACKET_V2:
2266 off = ph.h2->tp_mac;
2267 break;
2268 default:
2269 off = ph.h1->tp_mac;
2270 break;
2271 }
2272 }
2273 if (unlikely((off < off_min) || (off_max < off)))
2274 return -EINVAL;
2275 data = ph.raw + off;
2276 } else {
2277 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2278 }
69e3c75f
JB		 2279 to_write = tp_len;
2280
2281 if (sock->type == SOCK_DGRAM) {
2282 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2283 NULL, tp_len);
2284 if (unlikely(err < 0))
2285 return -EINVAL;
40d4e3df 2286 } else if (dev->hard_header_len) {
9c707762 2287 if (ll_header_truncated(dev, tp_len))
69e3c75f 2288 return -EINVAL;
69e3c75f
JB		 2289
2290 skb_push(skb, dev->hard_header_len);
2291 err = skb_store_bits(skb, 0, data,
2292 dev->hard_header_len);
2293 if (unlikely(err))
2294 return err;
2295
2296 data += dev->hard_header_len;
2297 to_write -= dev->hard_header_len;
2298 }
2299
69e3c75f
JB		 2300 offset = offset_in_page(data);
2301 len_max = PAGE_SIZE - offset;
2302 len = ((to_write > len_max) ? len_max : to_write);
2303
2304 skb->data_len = to_write;
2305 skb->len += to_write;
2306 skb->truesize += to_write;
2307 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2308
2309 while (likely(to_write)) {
2310 nr_frags = skb_shinfo(skb)->nr_frags;
2311
2312 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2313 pr_err("Packet exceed the number of skb frags(%lu)\n",
2314 MAX_SKB_FRAGS);
69e3c75f
JB		 2315 return -EFAULT;
2316 }
2317
0af55bb5
CG		 2318 page = pgv_to_page(data);
2319 data += len;
69e3c75f
JB		 2320 flush_dcache_page(page);
2321 get_page(page);
0af55bb5 2322 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2323 to_write -= len;
2324 offset = 0;
2325 len_max = PAGE_SIZE;
2326 len = ((to_write > len_max) ? len_max : to_write);
2327 }
2328
2329 return tp_len;
2330}
2331
2332static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2333{
69e3c75f
JB		 2334 struct sk_buff *skb;
2335 struct net_device *dev;
2336 __be16 proto;
09effa67 2337 int err, reserve = 0;
40d4e3df 2338 void *ph;
342dfc30 2339 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2340 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2341 int tp_len, size_max;
2342 unsigned char *addr;
2343 int len_sum = 0;
9e67030a 2344 int status = TP_STATUS_AVAILABLE;
ae641949 2345 int hlen, tlen;
69e3c75f 2346
69e3c75f
JB		 2347 mutex_lock(&po->pg_vec_lock);
2348
66e56cd4 2349 if (likely(saddr == NULL)) {
e40526cb 2350 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2351 proto = po->num;
2352 addr = NULL;
2353 } else {
2354 err = -EINVAL;
2355 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2356 goto out;
2357 if (msg->msg_namelen < (saddr->sll_halen
2358 + offsetof(struct sockaddr_ll,
2359 sll_addr)))
2360 goto out;
69e3c75f
JB		 2361 proto = saddr->sll_protocol;
2362 addr = saddr->sll_addr;
827d9780 2363 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2364 }
2365
69e3c75f
JB		 2366 err = -ENXIO;
2367 if (unlikely(dev == NULL))
2368 goto out;
69e3c75f
JB		 2369 err = -ENETDOWN;
2370 if (unlikely(!(dev->flags & IFF_UP)))
2371 goto out_put;
2372
52f1454f 2373 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2374 size_max = po->tx_ring.frame_size
b5dd884e 2375 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2376
09effa67
DM		 2377 if (size_max > dev->mtu + reserve)
2378 size_max = dev->mtu + reserve;
2379
69e3c75f
JB		 2380 do {
2381 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2382 TP_STATUS_SEND_REQUEST);
69e3c75f 2383 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2384 if (need_wait && need_resched())
2385 schedule();
69e3c75f
JB		 2386 continue;
2387 }
2388
2389 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2390 hlen = LL_RESERVED_SPACE(dev);
2391 tlen = dev->needed_tailroom;
69e3c75f 2392 skb = sock_alloc_send_skb(&po->sk,
ae641949 2393 hlen + tlen + sizeof(struct sockaddr_ll),
fbf33a28 2394 !need_wait, &err);
69e3c75f 2395
fbf33a28
KM		 2396 if (unlikely(skb == NULL)) {
2397 /* we assume the socket was initially writeable ... */
2398 if (likely(len_sum > 0))
2399 err = len_sum;
69e3c75f 2400 goto out_status;
fbf33a28 2401 }
69e3c75f 2402 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f 2403 addr, hlen);
dbd46ab4
AD		 2404 if (likely(tp_len >= 0) &&
2405 tp_len > dev->mtu + dev->hard_header_len) {
52f1454f
DB		 2406 struct ethhdr *ehdr;
2407 /* Earlier code assumed this would be a VLAN pkt,
2408 * double-check this now that we have the actual
2409 * packet in hand.
2410 */
69e3c75f 2411
52f1454f
DB		 2412 skb_reset_mac_header(skb);
2413 ehdr = eth_hdr(skb);
2414 if (ehdr->h_proto != htons(ETH_P_8021Q))
2415 tp_len = -EMSGSIZE;
2416 }
69e3c75f
JB		 2417 if (unlikely(tp_len < 0)) {
2418 if (po->tp_loss) {
2419 __packet_set_status(po, ph,
2420 TP_STATUS_AVAILABLE);
2421 packet_increment_head(&po->tx_ring);
2422 kfree_skb(skb);
2423 continue;
2424 } else {
2425 status = TP_STATUS_WRONG_FORMAT;
2426 err = tp_len;
2427 goto out_status;
2428 }
2429 }
2430
0fd5d57b
DB		 2431 packet_pick_tx_queue(dev, skb);
2432
69e3c75f
JB		 2433 skb->destructor = tpacket_destruct_skb;
2434 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2435 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2436
2437 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2438 err = po->xmit(skb);
eb70df13
JP		 2439 if (unlikely(err > 0)) {
2440 err = net_xmit_errno(err);
2441 if (err && __packet_get_status(po, ph) ==
2442 TP_STATUS_AVAILABLE) {
2443 /* skb was destructed already */
2444 skb = NULL;
2445 goto out_status;
2446 }
2447 /*
2448 * skb was dropped but not destructed yet;
2449 * let's treat it like congestion or err < 0
2450 */
2451 err = 0;
2452 }
69e3c75f
JB		 2453 packet_increment_head(&po->tx_ring);
2454 len_sum += tp_len;
b0138408
DB		 2455 } while (likely((ph != NULL) ||
2456 /* Note: packet_read_pending() might be slow if we have
2457 * to call it as it's per_cpu variable, but in fast-path
2458 * we already short-circuit the loop with the first
2459 * condition, and luckily don't have to go that path
2460 * anyway.
2461 */
2462 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2463
2464 err = len_sum;
2465 goto out_put;
2466
69e3c75f
JB		 2467out_status:
2468 __packet_set_status(po, ph, status);
2469 kfree_skb(skb);
2470out_put:
e40526cb 2471 dev_put(dev);
69e3c75f
JB		 2472out:
2473 mutex_unlock(&po->pg_vec_lock);
2474 return err;
2475}
69e3c75f 2476
eea49cc9
OJ		 2477static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2478 size_t reserve, size_t len,
2479 size_t linear, int noblock,
2480 int *err)
bfd5f4a3
SS		 2481{
2482 struct sk_buff *skb;
2483
2484 /* Under a page? Don't bother with paged skb. */
2485 if (prepad + len < PAGE_SIZE || !linear)
2486 linear = len;
2487
2488 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2489 err, 0);
bfd5f4a3
SS		 2490 if (!skb)
2491 return NULL;
2492
2493 skb_reserve(skb, reserve);
2494 skb_put(skb, linear);
2495 skb->data_len = len - linear;
2496 skb->len += len - linear;
2497
2498 return skb;
2499}
2500
d346a3fa 2501static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2502{
2503 struct sock *sk = sock->sk;
342dfc30 2504 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2505 struct sk_buff *skb;
2506 struct net_device *dev;
0e11c91e 2507 __be16 proto;
1da177e4 2508 unsigned char *addr;
827d9780 2509 int err, reserve = 0;
bfd5f4a3
SS		 2510 struct virtio_net_hdr vnet_hdr = { 0 };
2511 int offset = 0;
2512 int vnet_hdr_len;
2513 struct packet_sock *po = pkt_sk(sk);
2514 unsigned short gso_type = 0;
ae641949 2515 int hlen, tlen;
3bdc0eba 2516 int extra_len = 0;
8feb2fb2 2517 ssize_t n;
1da177e4
LT		 2518
2519 /*
1ce4f28b 2520 * Get and verify the address.
1da177e4 2521 */
1ce4f28b 2522
66e56cd4 2523 if (likely(saddr == NULL)) {
e40526cb 2524 dev = packet_cached_dev_get(po);
1da177e4
LT		 2525 proto = po->num;
2526 addr = NULL;
2527 } else {
2528 err = -EINVAL;
2529 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2530 goto out;
0fb375fb
EB		 2531 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2532 goto out;
1da177e4
LT		 2533 proto = saddr->sll_protocol;
2534 addr = saddr->sll_addr;
827d9780 2535 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2536 }
2537
1da177e4 2538 err = -ENXIO;
e40526cb 2539 if (unlikely(dev == NULL))
1da177e4 2540 goto out_unlock;
d5e76b0a 2541 err = -ENETDOWN;
e40526cb 2542 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2543 goto out_unlock;
2544
e40526cb
DB		 2545 if (sock->type == SOCK_RAW)
2546 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2547 if (po->has_vnet_hdr) {
2548 vnet_hdr_len = sizeof(vnet_hdr);
2549
2550 err = -EINVAL;
2551 if (len < vnet_hdr_len)
2552 goto out_unlock;
2553
2554 len -= vnet_hdr_len;
2555
8feb2fb2 2556 err = -EFAULT;
c0371da6 2557 n = copy_from_iter(&vnet_hdr, vnet_hdr_len, &msg->msg_iter);
8feb2fb2 2558 if (n != vnet_hdr_len)
bfd5f4a3
SS		 2559 goto out_unlock;
2560
2561 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
dc9e5153
MT		 2562 (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
2563 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
2564 __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
2565 vnet_hdr.hdr_len = __cpu_to_virtio16(false,
2566 __virtio16_to_cpu(false, vnet_hdr.csum_start) +
2567 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
bfd5f4a3
SS		 2568
2569 err = -EINVAL;
dc9e5153 2570 if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
bfd5f4a3
SS		 2571 goto out_unlock;
2572
2573 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2574 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2575 case VIRTIO_NET_HDR_GSO_TCPV4:
2576 gso_type = SKB_GSO_TCPV4;
2577 break;
2578 case VIRTIO_NET_HDR_GSO_TCPV6:
2579 gso_type = SKB_GSO_TCPV6;
2580 break;
2581 case VIRTIO_NET_HDR_GSO_UDP:
2582 gso_type = SKB_GSO_UDP;
2583 break;
2584 default:
2585 goto out_unlock;
2586 }
2587
2588 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2589 gso_type |= SKB_GSO_TCP_ECN;
2590
2591 if (vnet_hdr.gso_size == 0)
2592 goto out_unlock;
2593
2594 }
2595 }
2596
3bdc0eba
BG		 2597 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2598 if (!netif_supports_nofcs(dev)) {
2599 err = -EPROTONOSUPPORT;
2600 goto out_unlock;
2601 }
2602 extra_len = 4; /* We're doing our own CRC */
2603 }
2604
1da177e4 2605 err = -EMSGSIZE;
3bdc0eba 2606 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2607 goto out_unlock;
2608
bfd5f4a3 2609 err = -ENOBUFS;
ae641949
HX		 2610 hlen = LL_RESERVED_SPACE(dev);
2611 tlen = dev->needed_tailroom;
dc9e5153
MT		 2612 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2613 __virtio16_to_cpu(false, vnet_hdr.hdr_len),
bfd5f4a3 2614 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2615 if (skb == NULL)
1da177e4
LT		 2616 goto out_unlock;
2617
bfd5f4a3 2618 skb_set_network_header(skb, reserve);
1da177e4 2619
0c4e8581 2620 err = -EINVAL;
9c707762
WB		 2621 if (sock->type == SOCK_DGRAM) {
2622 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
46d2cfb1 2623 if (unlikely(offset < 0))
9c707762
WB		 2624 goto out_free;
2625 } else {
2626 if (ll_header_truncated(dev, len))
2627 goto out_free;
2628 }
1da177e4
LT		 2629
2630 /* Returns -EFAULT on error */
c0371da6 2631 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
1da177e4
LT		 2632 if (err)
2633 goto out_free;
bf84a010
DB		 2634
2635 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2636
3bdc0eba 2637 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2638 /* Earlier code assumed this would be a VLAN pkt,
2639 * double-check this now that we have the actual
2640 * packet in hand.
2641 */
2642 struct ethhdr *ehdr;
2643 skb_reset_mac_header(skb);
2644 ehdr = eth_hdr(skb);
2645 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2646 err = -EMSGSIZE;
2647 goto out_free;
2648 }
57f89bfa
BG		 2649 }
2650
09effa67
DM		 2651 skb->protocol = proto;
2652 skb->dev = dev;
1da177e4 2653 skb->priority = sk->sk_priority;
2d37a186 2654 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2655
2656 packet_pick_tx_queue(dev, skb);
1da177e4 2657
bfd5f4a3
SS		 2658 if (po->has_vnet_hdr) {
2659 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
dc9e5153
MT		 2660 u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
2661 u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
2662 if (!skb_partial_csum_set(skb, s, o)) {
bfd5f4a3
SS		 2663 err = -EINVAL;
2664 goto out_free;
2665 }
2666 }
2667
dc9e5153
MT		 2668 skb_shinfo(skb)->gso_size =
2669 __virtio16_to_cpu(false, vnet_hdr.gso_size);
bfd5f4a3
SS		 2670 skb_shinfo(skb)->gso_type = gso_type;
2671
2672 /* Header must be checked, and gso_segs computed. */
2673 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2674 skb_shinfo(skb)->gso_segs = 0;
2675
2676 len += vnet_hdr_len;
2677 }
2678
d346a3fa
DB		 2679 if (!packet_use_direct_xmit(po))
2680 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2681 if (unlikely(extra_len == 4))
2682 skb->no_fcs = 1;
2683
d346a3fa 2684 err = po->xmit(skb);
1da177e4
LT		 2685 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2686 goto out_unlock;
2687
e40526cb 2688 dev_put(dev);
1da177e4 2689
40d4e3df 2690 return len;
1da177e4
LT		 2691
2692out_free:
2693 kfree_skb(skb);
2694out_unlock:
e40526cb 2695 if (dev)
1da177e4
LT		 2696 dev_put(dev);
2697out:
2698 return err;
2699}
2700
1b784140 2701static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
69e3c75f 2702{
69e3c75f
JB		 2703 struct sock *sk = sock->sk;
2704 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2705
69e3c75f
JB		 2706 if (po->tx_ring.pg_vec)
2707 return tpacket_snd(po, msg);
2708 else
69e3c75f
JB		 2709 return packet_snd(sock, msg, len);
2710}
2711
1da177e4
LT		 2712/*
2713 * Close a PACKET socket. This is fairly simple. We immediately go
2714 * to 'closed' state and remove our protocol entry in the device list.
2715 */
2716
2717static int packet_release(struct socket *sock)
2718{
2719 struct sock *sk = sock->sk;
2720 struct packet_sock *po;
d12d01d6 2721 struct net *net;
f6fb8f10 2722 union tpacket_req_u req_u;
1da177e4
LT		 2723
2724 if (!sk)
2725 return 0;
2726
3b1e0a65 2727 net = sock_net(sk);
1da177e4
LT		 2728 po = pkt_sk(sk);
2729
0fa7fa98 2730 mutex_lock(&net->packet.sklist_lock);
808f5114 2731 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2732 mutex_unlock(&net->packet.sklist_lock);
2733
2734 preempt_disable();
920de804 2735 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2736 preempt_enable();
1da177e4 2737
808f5114 2738 spin_lock(&po->bind_lock);
ce06b03e 2739 unregister_prot_hook(sk, false);
66e56cd4
DB		 2740 packet_cached_dev_reset(po);
2741
160ff18a
BG		 2742 if (po->prot_hook.dev) {
2743 dev_put(po->prot_hook.dev);
2744 po->prot_hook.dev = NULL;
2745 }
808f5114 2746 spin_unlock(&po->bind_lock);
1da177e4 2747
1da177e4 2748 packet_flush_mclist(sk);
1da177e4 2749
9665d5d6
PS		 2750 if (po->rx_ring.pg_vec) {
2751 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2752 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2753 }
69e3c75f 2754
9665d5d6
PS		 2755 if (po->tx_ring.pg_vec) {
2756 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2757 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2758 }
1da177e4 2759
dc99f600
DM		 2760 fanout_release(sk);
2761
808f5114 2762 synchronize_net();
1da177e4
LT		 2763 /*
2764 * Now the socket is dead. No more input will appear.
2765 */
1da177e4
LT		 2766 sock_orphan(sk);
2767 sock->sk = NULL;
2768
2769 /* Purge queues */
2770
2771 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2772 packet_free_pending(po);
17ab56a2 2773 sk_refcnt_debug_release(sk);
1da177e4
LT		 2774
2775 sock_put(sk);
2776 return 0;
2777}
2778
2779/*
2780 * Attach a packet hook.
2781 */
2782
902fefb8 2783static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2784{
2785 struct packet_sock *po = pkt_sk(sk);
158cd4af 2786 struct net_device *dev_curr;
902fefb8
DB		 2787 __be16 proto_curr;
2788 bool need_rehook;
dc99f600 2789
aef950b4
WY		 2790 if (po->fanout) {
2791 if (dev)
2792 dev_put(dev);
2793
dc99f600 2794 return -EINVAL;
aef950b4 2795 }
1da177e4
LT		 2796
2797 lock_sock(sk);
1da177e4 2798 spin_lock(&po->bind_lock);
66e56cd4 2799
902fefb8
DB		 2800 proto_curr = po->prot_hook.type;
2801 dev_curr = po->prot_hook.dev;
2802
2803 need_rehook = proto_curr != proto || dev_curr != dev;
2804
2805 if (need_rehook) {
2806 unregister_prot_hook(sk, true);
1da177e4 2807
902fefb8
DB		 2808 po->num = proto;
2809 po->prot_hook.type = proto;
902fefb8
DB		 2810 po->prot_hook.dev = dev;
2811
2812 po->ifindex = dev ? dev->ifindex : 0;
2813 packet_cached_dev_assign(po, dev);
2814 }
158cd4af
LW		 2815 if (dev_curr)
2816 dev_put(dev_curr);
66e56cd4 2817
902fefb8 2818 if (proto == 0 || !need_rehook)
1da177e4
LT		 2819 goto out_unlock;
2820
be85d4ad 2821 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2822 register_prot_hook(sk);
be85d4ad
UT		 2823 } else {
2824 sk->sk_err = ENETDOWN;
2825 if (!sock_flag(sk, SOCK_DEAD))
2826 sk->sk_error_report(sk);
1da177e4
LT		 2827 }
2828
2829out_unlock:
2830 spin_unlock(&po->bind_lock);
2831 release_sock(sk);
2832 return 0;
2833}
2834
2835/*
2836 * Bind a packet socket to a device
2837 */
2838
40d4e3df
ED		 2839static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2840 int addr_len)
1da177e4 2841{
40d4e3df 2842 struct sock *sk = sock->sk;
1da177e4
LT		 2843 char name[15];
2844 struct net_device *dev;
2845 int err = -ENODEV;
1ce4f28b 2846
1da177e4
LT		 2847 /*
2848 * Check legality
2849 */
1ce4f28b 2850
8ae55f04 2851 if (addr_len != sizeof(struct sockaddr))
1da177e4 2852 return -EINVAL;
40d4e3df 2853 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2854
3b1e0a65 2855 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2856 if (dev)
1da177e4 2857 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2858 return err;
2859}
1da177e4
LT		 2860
2861static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2862{
40d4e3df
ED		 2863 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2864 struct sock *sk = sock->sk;
1da177e4
LT		 2865 struct net_device *dev = NULL;
2866 int err;
2867
2868
2869 /*
2870 * Check legality
2871 */
1ce4f28b 2872
1da177e4
LT		 2873 if (addr_len < sizeof(struct sockaddr_ll))
2874 return -EINVAL;
2875 if (sll->sll_family != AF_PACKET)
2876 return -EINVAL;
2877
2878 if (sll->sll_ifindex) {
2879 err = -ENODEV;
3b1e0a65 2880 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2881 if (dev == NULL)
2882 goto out;
2883 }
2884 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2885
2886out:
2887 return err;
2888}
2889
2890static struct proto packet_proto = {
2891 .name = "PACKET",
2892 .owner = THIS_MODULE,
2893 .obj_size = sizeof(struct packet_sock),
2894};
2895
2896/*
1ce4f28b 2897 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2898 */
2899
3f378b68
EP		 2900static int packet_create(struct net *net, struct socket *sock, int protocol,
2901 int kern)
1da177e4
LT		 2902{
2903 struct sock *sk;
2904 struct packet_sock *po;
0e11c91e 2905 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2906 int err;
2907
df008c91 2908 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2909 return -EPERM;
be02097c
DM		 2910 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2911 sock->type != SOCK_PACKET)
1da177e4
LT		 2912 return -ESOCKTNOSUPPORT;
2913
2914 sock->state = SS_UNCONNECTED;
2915
2916 err = -ENOBUFS;
11aa9c28 2917 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
1da177e4
LT		 2918 if (sk == NULL)
2919 goto out;
2920
2921 sock->ops = &packet_ops;
1da177e4
LT		 2922 if (sock->type == SOCK_PACKET)
2923 sock->ops = &packet_ops_spkt;
be02097c 2924
1da177e4
LT		 2925 sock_init_data(sock, sk);
2926
2927 po = pkt_sk(sk);
2928 sk->sk_family = PF_PACKET;
0e11c91e 2929 po->num = proto;
d346a3fa 2930 po->xmit = dev_queue_xmit;
66e56cd4 2931
b0138408
DB		 2932 err = packet_alloc_pending(po);
2933 if (err)
2934 goto out2;
2935
66e56cd4 2936 packet_cached_dev_reset(po);
1da177e4
LT		 2937
2938 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2939 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2940
2941 /*
2942 * Attach a protocol block
2943 */
2944
2945 spin_lock_init(&po->bind_lock);
905db440 2946 mutex_init(&po->pg_vec_lock);
0648ab70 2947 po->rollover = NULL;
1da177e4 2948 po->prot_hook.func = packet_rcv;
be02097c 2949
1da177e4
LT		 2950 if (sock->type == SOCK_PACKET)
2951 po->prot_hook.func = packet_rcv_spkt;
be02097c 2952
1da177e4
LT		 2953 po->prot_hook.af_packet_priv = sk;
2954
0e11c91e
AV		 2955 if (proto) {
2956 po->prot_hook.type = proto;
ce06b03e 2957 register_prot_hook(sk);
1da177e4
LT		 2958 }
2959
0fa7fa98 2960 mutex_lock(&net->packet.sklist_lock);
808f5114 2961 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2962 mutex_unlock(&net->packet.sklist_lock);
2963
2964 preempt_disable();
3680453c 2965 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2966 preempt_enable();
808f5114 2967
40d4e3df 2968 return 0;
b0138408
DB		 2969out2:
2970 sk_free(sk);
1da177e4
LT		 2971out:
2972 return err;
2973}
2974
2975/*
2976 * Pull a packet from our receive queue and hand it to the user.
2977 * If necessary we block.
2978 */
2979
1b784140
YX		 2980static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2981 int flags)
1da177e4
LT		 2982{
2983 struct sock *sk = sock->sk;
2984 struct sk_buff *skb;
2985 int copied, err;
bfd5f4a3 2986 int vnet_hdr_len = 0;
2472d761 2987 unsigned int origlen = 0;
1da177e4
LT		 2988
2989 err = -EINVAL;
ed85b565 2990 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2991 goto out;
2992
2993#if 0
2994 /* What error should we return now? EUNATTACH? */
2995 if (pkt_sk(sk)->ifindex < 0)
2996 return -ENODEV;
2997#endif
2998
ed85b565 2999 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 3000 err = sock_recv_errqueue(sk, msg, len,
3001 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 3002 goto out;
3003 }
3004
1da177e4
LT		 3005 /*
3006 * Call the generic datagram receiver. This handles all sorts
3007 * of horrible races and re-entrancy so we can forget about it
3008 * in the protocol layers.
3009 *
3010 * Now it will return ENETDOWN, if device have just gone down,
3011 * but then it will block.
3012 */
3013
40d4e3df 3014 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 3015
3016 /*
1ce4f28b 3017 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 3018 * handles the blocking we don't see and worry about blocking
3019 * retries.
3020 */
3021
8ae55f04 3022 if (skb == NULL)
1da177e4
LT		 3023 goto out;
3024
2ccdbaa6
WB		 3025 if (pkt_sk(sk)->pressure)
3026 packet_rcv_has_room(pkt_sk(sk), NULL);
3027
bfd5f4a3
SS		 3028 if (pkt_sk(sk)->has_vnet_hdr) {
3029 struct virtio_net_hdr vnet_hdr = { 0 };
3030
3031 err = -EINVAL;
3032 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 3033 if (len < vnet_hdr_len)
bfd5f4a3
SS		 3034 goto out_free;
3035
1f18b717
MK		 3036 len -= vnet_hdr_len;
3037
bfd5f4a3
SS		 3038 if (skb_is_gso(skb)) {
3039 struct skb_shared_info *sinfo = skb_shinfo(skb);
3040
3041 /* This is a hint as to how much should be linear. */
dc9e5153
MT		 3042 vnet_hdr.hdr_len =
3043 __cpu_to_virtio16(false, skb_headlen(skb));
3044 vnet_hdr.gso_size =
3045 __cpu_to_virtio16(false, sinfo->gso_size);
bfd5f4a3
SS		 3046 if (sinfo->gso_type & SKB_GSO_TCPV4)
3047 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
3048 else if (sinfo->gso_type & SKB_GSO_TCPV6)
3049 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
3050 else if (sinfo->gso_type & SKB_GSO_UDP)
3051 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
3052 else if (sinfo->gso_type & SKB_GSO_FCOE)
3053 goto out_free;
3054 else
3055 BUG();
3056 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
3057 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
3058 } else
3059 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
3060
3061 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3062 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
dc9e5153
MT		 3063 vnet_hdr.csum_start = __cpu_to_virtio16(false,
3064 skb_checksum_start_offset(skb));
3065 vnet_hdr.csum_offset = __cpu_to_virtio16(false,
3066 skb->csum_offset);
10a8d94a
JW		 3067 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3068 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 3069 } /* else everything is zero */
3070
7eab8d9e 3071 err = memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_len);
bfd5f4a3
SS		 3072 if (err < 0)
3073 goto out_free;
3074 }
3075
f3d33426
HFS		 3076 /* You lose any data beyond the buffer you gave. If it worries
3077 * a user program they can ask the device for its MTU
3078 * anyway.
1da177e4 3079 */
1da177e4 3080 copied = skb->len;
40d4e3df
ED		 3081 if (copied > len) {
3082 copied = len;
3083 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 3084 }
3085
51f3d02b 3086 err = skb_copy_datagram_msg(skb, 0, msg, copied);
1da177e4
LT		 3087 if (err)
3088 goto out_free;
3089
2472d761
EB		 3090 if (sock->type != SOCK_PACKET) {
3091 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3092
3093 /* Original length was stored in sockaddr_ll fields */
3094 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3095 sll->sll_family = AF_PACKET;
3096 sll->sll_protocol = skb->protocol;
3097 }
3098
3b885787 3099 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 3100
f3d33426
HFS		 3101 if (msg->msg_name) {
3102 /* If the address length field is there to be filled
3103 * in, we fill it in now.
3104 */
3105 if (sock->type == SOCK_PACKET) {
342dfc30 3106 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 3107 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3108 } else {
3109 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2472d761 3110
f3d33426
HFS		 3111 msg->msg_namelen = sll->sll_halen +
3112 offsetof(struct sockaddr_ll, sll_addr);
3113 }
ffbc6111
HX		 3114 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3115 msg->msg_namelen);
f3d33426 3116 }
1da177e4 3117
8dc41944 3118 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 3119 struct tpacket_auxdata aux;
3120
3121 aux.tp_status = TP_STATUS_USER;
3122 if (skb->ip_summed == CHECKSUM_PARTIAL)
3123 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 3124 else if (skb->pkt_type != PACKET_OUTGOING &&
3125 (skb->ip_summed == CHECKSUM_COMPLETE ||
3126 skb_csum_unnecessary(skb)))
3127 aux.tp_status |= TP_STATUS_CSUM_VALID;
3128
2472d761 3129 aux.tp_len = origlen;
ffbc6111
HX		 3130 aux.tp_snaplen = skb->len;
3131 aux.tp_mac = 0;
bbe735e4 3132 aux.tp_net = skb_network_offset(skb);
df8a39de
JP		 3133 if (skb_vlan_tag_present(skb)) {
3134 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 3135 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3136 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 3137 } else {
3138 aux.tp_vlan_tci = 0;
a0cdfcf3 3139 aux.tp_vlan_tpid = 0;
a3bcc23e 3140 }
ffbc6111 3141 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3142 }
3143
1da177e4
LT		 3144 /*
3145 * Free or return the buffer as appropriate. Again this
3146 * hides all the races and re-entrancy issues from us.
3147 */
bfd5f4a3 3148 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3149
3150out_free:
3151 skb_free_datagram(sk, skb);
3152out:
3153 return err;
3154}
3155
1da177e4
LT		 3156static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3157 int *uaddr_len, int peer)
3158{
3159 struct net_device *dev;
3160 struct sock *sk = sock->sk;
3161
3162 if (peer)
3163 return -EOPNOTSUPP;
3164
3165 uaddr->sa_family = AF_PACKET;
2dc85bf3 3166 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3167 rcu_read_lock();
3168 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3169 if (dev)
2dc85bf3 3170 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3171 rcu_read_unlock();
1da177e4
LT		 3172 *uaddr_len = sizeof(*uaddr);
3173
3174 return 0;
3175}
1da177e4
LT		 3176
3177static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3178 int *uaddr_len, int peer)
3179{
3180 struct net_device *dev;
3181 struct sock *sk = sock->sk;
3182 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3183 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3184
3185 if (peer)
3186 return -EOPNOTSUPP;
3187
3188 sll->sll_family = AF_PACKET;
3189 sll->sll_ifindex = po->ifindex;
3190 sll->sll_protocol = po->num;
67286640 3191 sll->sll_pkttype = 0;
654d1f8a
ED		 3192 rcu_read_lock();
3193 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3194 if (dev) {
3195 sll->sll_hatype = dev->type;
3196 sll->sll_halen = dev->addr_len;
3197 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3198 } else {
3199 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3200 sll->sll_halen = 0;
3201 }
654d1f8a 3202 rcu_read_unlock();
0fb375fb 3203 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3204
3205 return 0;
3206}
3207
2aeb0b88
WC		 3208static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3209 int what)
1da177e4
LT		 3210{
3211 switch (i->type) {
3212 case PACKET_MR_MULTICAST:
1162563f
JP		 3213 if (i->alen != dev->addr_len)
3214 return -EINVAL;
1da177e4 3215 if (what > 0)
22bedad3 3216 return dev_mc_add(dev, i->addr);
1da177e4 3217 else
22bedad3 3218 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3219 break;
3220 case PACKET_MR_PROMISC:
2aeb0b88 3221 return dev_set_promiscuity(dev, what);
1da177e4 3222 case PACKET_MR_ALLMULTI:
2aeb0b88 3223 return dev_set_allmulti(dev, what);
d95ed927 3224 case PACKET_MR_UNICAST:
1162563f
JP		 3225 if (i->alen != dev->addr_len)
3226 return -EINVAL;
d95ed927 3227 if (what > 0)
a748ee24 3228 return dev_uc_add(dev, i->addr);
d95ed927 3229 else
a748ee24 3230 return dev_uc_del(dev, i->addr);
d95ed927 3231 break;
40d4e3df
ED		 3232 default:
3233 break;
1da177e4 3234 }
2aeb0b88 3235 return 0;
1da177e4
LT		 3236}
3237
82f17091
FR		 3238static void packet_dev_mclist_delete(struct net_device *dev,
3239 struct packet_mclist **mlp)
1da177e4 3240{
82f17091
FR		 3241 struct packet_mclist *ml;
3242
3243 while ((ml = *mlp) != NULL) {
3244 if (ml->ifindex == dev->ifindex) {
3245 packet_dev_mc(dev, ml, -1);
3246 *mlp = ml->next;
3247 kfree(ml);
3248 } else
3249 mlp = &ml->next;
1da177e4
LT		 3250 }
3251}
3252
0fb375fb 3253static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3254{
3255 struct packet_sock *po = pkt_sk(sk);
3256 struct packet_mclist *ml, *i;
3257 struct net_device *dev;
3258 int err;
3259
3260 rtnl_lock();
3261
3262 err = -ENODEV;
3b1e0a65 3263 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3264 if (!dev)
3265 goto done;
3266
3267 err = -EINVAL;
1162563f 3268 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3269 goto done;
3270
3271 err = -ENOBUFS;
8b3a7005 3272 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3273 if (i == NULL)
3274 goto done;
3275
3276 err = 0;
3277 for (ml = po->mclist; ml; ml = ml->next) {
3278 if (ml->ifindex == mreq->mr_ifindex &&
3279 ml->type == mreq->mr_type &&
3280 ml->alen == mreq->mr_alen &&
3281 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3282 ml->count++;
3283 /* Free the new element ... */
3284 kfree(i);
3285 goto done;
3286 }
3287 }
3288
3289 i->type = mreq->mr_type;
3290 i->ifindex = mreq->mr_ifindex;
3291 i->alen = mreq->mr_alen;
3292 memcpy(i->addr, mreq->mr_address, i->alen);
3293 i->count = 1;
3294 i->next = po->mclist;
3295 po->mclist = i;
2aeb0b88
WC		 3296 err = packet_dev_mc(dev, i, 1);
3297 if (err) {
3298 po->mclist = i->next;
3299 kfree(i);
3300 }
1da177e4
LT		 3301
3302done:
3303 rtnl_unlock();
3304 return err;
3305}
3306
0fb375fb 3307static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3308{
3309 struct packet_mclist *ml, **mlp;
3310
3311 rtnl_lock();
3312
3313 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3314 if (ml->ifindex == mreq->mr_ifindex &&
3315 ml->type == mreq->mr_type &&
3316 ml->alen == mreq->mr_alen &&
3317 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3318 if (--ml->count == 0) {
3319 struct net_device *dev;
3320 *mlp = ml->next;
ad959e76
ED		 3321 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3322 if (dev)
1da177e4 3323 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3324 kfree(ml);
3325 }
82f17091 3326 break;
1da177e4
LT		 3327 }
3328 }
3329 rtnl_unlock();
82f17091 3330 return 0;
1da177e4
LT		 3331}
3332
3333static void packet_flush_mclist(struct sock *sk)
3334{
3335 struct packet_sock *po = pkt_sk(sk);
3336 struct packet_mclist *ml;
3337
3338 if (!po->mclist)
3339 return;
3340
3341 rtnl_lock();
3342 while ((ml = po->mclist) != NULL) {
3343 struct net_device *dev;
3344
3345 po->mclist = ml->next;
ad959e76
ED		 3346 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3347 if (dev != NULL)
1da177e4 3348 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3349 kfree(ml);
3350 }
3351 rtnl_unlock();
3352}
1da177e4
LT		 3353
3354static int
b7058842 3355packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3356{
3357 struct sock *sk = sock->sk;
8dc41944 3358 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3359 int ret;
3360
3361 if (level != SOL_PACKET)
3362 return -ENOPROTOOPT;
3363
69e3c75f 3364 switch (optname) {
1ce4f28b 3365 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3366 case PACKET_DROP_MEMBERSHIP:
3367 {
0fb375fb
EB		 3368 struct packet_mreq_max mreq;
3369 int len = optlen;
3370 memset(&mreq, 0, sizeof(mreq));
3371 if (len < sizeof(struct packet_mreq))
1da177e4 3372 return -EINVAL;
0fb375fb
EB		 3373 if (len > sizeof(mreq))
3374 len = sizeof(mreq);
40d4e3df 3375 if (copy_from_user(&mreq, optval, len))
1da177e4 3376 return -EFAULT;
0fb375fb
EB		 3377 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3378 return -EINVAL;
1da177e4
LT		 3379 if (optname == PACKET_ADD_MEMBERSHIP)
3380 ret = packet_mc_add(sk, &mreq);
3381 else
3382 ret = packet_mc_drop(sk, &mreq);
3383 return ret;
3384 }
a2efcfa0 3385
1da177e4 3386 case PACKET_RX_RING:
69e3c75f 3387 case PACKET_TX_RING:
1da177e4 3388 {
f6fb8f10 3389 union tpacket_req_u req_u;
3390 int len;
1da177e4 3391
f6fb8f10 3392 switch (po->tp_version) {
3393 case TPACKET_V1:
3394 case TPACKET_V2:
3395 len = sizeof(req_u.req);
3396 break;
3397 case TPACKET_V3:
3398 default:
3399 len = sizeof(req_u.req3);
3400 break;
3401 }
3402 if (optlen < len)
1da177e4 3403 return -EINVAL;
bfd5f4a3
SS		 3404 if (pkt_sk(sk)->has_vnet_hdr)
3405 return -EINVAL;
f6fb8f10 3406 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3407 return -EFAULT;
f6fb8f10 3408 return packet_set_ring(sk, &req_u, 0,
3409 optname == PACKET_TX_RING);
1da177e4
LT		 3410 }
3411 case PACKET_COPY_THRESH:
3412 {
3413 int val;
3414
40d4e3df 3415 if (optlen != sizeof(val))
1da177e4 3416 return -EINVAL;
40d4e3df 3417 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3418 return -EFAULT;
3419
3420 pkt_sk(sk)->copy_thresh = val;
3421 return 0;
3422 }
bbd6ef87
PM		 3423 case PACKET_VERSION:
3424 {
3425 int val;
3426
3427 if (optlen != sizeof(val))
3428 return -EINVAL;
69e3c75f 3429 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3430 return -EBUSY;
3431 if (copy_from_user(&val, optval, sizeof(val)))
3432 return -EFAULT;
3433 switch (val) {
3434 case TPACKET_V1:
3435 case TPACKET_V2:
f6fb8f10 3436 case TPACKET_V3:
bbd6ef87
PM		 3437 po->tp_version = val;
3438 return 0;
3439 default:
3440 return -EINVAL;
3441 }
3442 }
8913336a
PM		 3443 case PACKET_RESERVE:
3444 {
3445 unsigned int val;
3446
3447 if (optlen != sizeof(val))
3448 return -EINVAL;
69e3c75f 3449 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3450 return -EBUSY;
3451 if (copy_from_user(&val, optval, sizeof(val)))
3452 return -EFAULT;
3453 po->tp_reserve = val;
3454 return 0;
3455 }
69e3c75f
JB		 3456 case PACKET_LOSS:
3457 {
3458 unsigned int val;
3459
3460 if (optlen != sizeof(val))
3461 return -EINVAL;
3462 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3463 return -EBUSY;
3464 if (copy_from_user(&val, optval, sizeof(val)))
3465 return -EFAULT;
3466 po->tp_loss = !!val;
3467 return 0;
3468 }
8dc41944
HX		 3469 case PACKET_AUXDATA:
3470 {
3471 int val;
3472
3473 if (optlen < sizeof(val))
3474 return -EINVAL;
3475 if (copy_from_user(&val, optval, sizeof(val)))
3476 return -EFAULT;
3477
3478 po->auxdata = !!val;
3479 return 0;
3480 }
80feaacb
PWJ		 3481 case PACKET_ORIGDEV:
3482 {
3483 int val;
3484
3485 if (optlen < sizeof(val))
3486 return -EINVAL;
3487 if (copy_from_user(&val, optval, sizeof(val)))
3488 return -EFAULT;
3489
3490 po->origdev = !!val;
3491 return 0;
3492 }
bfd5f4a3
SS		 3493 case PACKET_VNET_HDR:
3494 {
3495 int val;
3496
3497 if (sock->type != SOCK_RAW)
3498 return -EINVAL;
3499 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3500 return -EBUSY;
3501 if (optlen < sizeof(val))
3502 return -EINVAL;
3503 if (copy_from_user(&val, optval, sizeof(val)))
3504 return -EFAULT;
3505
3506 po->has_vnet_hdr = !!val;
3507 return 0;
3508 }
614f60fa
SM		 3509 case PACKET_TIMESTAMP:
3510 {
3511 int val;
3512
3513 if (optlen != sizeof(val))
3514 return -EINVAL;
3515 if (copy_from_user(&val, optval, sizeof(val)))
3516 return -EFAULT;
3517
3518 po->tp_tstamp = val;
3519 return 0;
3520 }
dc99f600
DM		 3521 case PACKET_FANOUT:
3522 {
3523 int val;
3524
3525 if (optlen != sizeof(val))
3526 return -EINVAL;
3527 if (copy_from_user(&val, optval, sizeof(val)))
3528 return -EFAULT;
3529
3530 return fanout_add(sk, val & 0xffff, val >> 16);
3531 }
5920cd3a
PC		 3532 case PACKET_TX_HAS_OFF:
3533 {
3534 unsigned int val;
3535
3536 if (optlen != sizeof(val))
3537 return -EINVAL;
3538 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3539 return -EBUSY;
3540 if (copy_from_user(&val, optval, sizeof(val)))
3541 return -EFAULT;
3542 po->tp_tx_has_off = !!val;
3543 return 0;
3544 }
d346a3fa
DB		 3545 case PACKET_QDISC_BYPASS:
3546 {
3547 int val;
3548
3549 if (optlen != sizeof(val))
3550 return -EINVAL;
3551 if (copy_from_user(&val, optval, sizeof(val)))
3552 return -EFAULT;
3553
3554 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3555 return 0;
3556 }
1da177e4
LT		 3557 default:
3558 return -ENOPROTOOPT;
3559 }
3560}
3561
3562static int packet_getsockopt(struct socket *sock, int level, int optname,
3563 char __user *optval, int __user *optlen)
3564{
3565 int len;
c06fff6e 3566 int val, lv = sizeof(val);
1da177e4
LT		 3567 struct sock *sk = sock->sk;
3568 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3569 void *data = &val;
ee80fbf3 3570 union tpacket_stats_u st;
a9b63918 3571 struct tpacket_rollover_stats rstats;
1da177e4
LT		 3572
3573 if (level != SOL_PACKET)
3574 return -ENOPROTOOPT;
3575
8ae55f04
KK		 3576 if (get_user(len, optlen))
3577 return -EFAULT;
1da177e4
LT		 3578
3579 if (len < 0)
3580 return -EINVAL;
1ce4f28b 3581
69e3c75f 3582 switch (optname) {
1da177e4 3583 case PACKET_STATISTICS:
1da177e4 3584 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3585 memcpy(&st, &po->stats, sizeof(st));
3586 memset(&po->stats, 0, sizeof(po->stats));
3587 spin_unlock_bh(&sk->sk_receive_queue.lock);
3588
f6fb8f10 3589 if (po->tp_version == TPACKET_V3) {
c06fff6e 3590 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3591 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3592 data = &st.stats3;
f6fb8f10 3593 } else {
c06fff6e 3594 lv = sizeof(struct tpacket_stats);
8bcdeaff 3595 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3596 data = &st.stats1;
f6fb8f10 3597 }
ee80fbf3 3598
8dc41944
HX		 3599 break;
3600 case PACKET_AUXDATA:
8dc41944 3601 val = po->auxdata;
80feaacb
PWJ		 3602 break;
3603 case PACKET_ORIGDEV:
80feaacb 3604 val = po->origdev;
bfd5f4a3
SS		 3605 break;
3606 case PACKET_VNET_HDR:
bfd5f4a3 3607 val = po->has_vnet_hdr;
1da177e4 3608 break;
bbd6ef87 3609 case PACKET_VERSION:
bbd6ef87 3610 val = po->tp_version;
bbd6ef87
PM		 3611 break;
3612 case PACKET_HDRLEN:
3613 if (len > sizeof(int))
3614 len = sizeof(int);
3615 if (copy_from_user(&val, optval, len))
3616 return -EFAULT;
3617 switch (val) {
3618 case TPACKET_V1:
3619 val = sizeof(struct tpacket_hdr);
3620 break;
3621 case TPACKET_V2:
3622 val = sizeof(struct tpacket2_hdr);
3623 break;
f6fb8f10 3624 case TPACKET_V3:
3625 val = sizeof(struct tpacket3_hdr);
3626 break;
bbd6ef87
PM		 3627 default:
3628 return -EINVAL;
3629 }
bbd6ef87 3630 break;
8913336a 3631 case PACKET_RESERVE:
8913336a 3632 val = po->tp_reserve;
8913336a 3633 break;
69e3c75f 3634 case PACKET_LOSS:
69e3c75f 3635 val = po->tp_loss;
69e3c75f 3636 break;
614f60fa 3637 case PACKET_TIMESTAMP:
614f60fa 3638 val = po->tp_tstamp;
614f60fa 3639 break;
dc99f600 3640 case PACKET_FANOUT:
dc99f600
DM		 3641 val = (po->fanout ?
3642 ((u32)po->fanout->id |
77f65ebd
WB		 3643 ((u32)po->fanout->type << 16) |
3644 ((u32)po->fanout->flags << 24)) :
dc99f600 3645 0);
dc99f600 3646 break;
a9b63918
WB		 3647 case PACKET_ROLLOVER_STATS:
3648 if (!po->rollover)
3649 return -EINVAL;
3650 rstats.tp_all = atomic_long_read(&po->rollover->num);
3651 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3652 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3653 data = &rstats;
3654 lv = sizeof(rstats);
3655 break;
5920cd3a
PC		 3656 case PACKET_TX_HAS_OFF:
3657 val = po->tp_tx_has_off;
3658 break;
d346a3fa
DB		 3659 case PACKET_QDISC_BYPASS:
3660 val = packet_use_direct_xmit(po);
3661 break;
1da177e4
LT		 3662 default:
3663 return -ENOPROTOOPT;
3664 }
3665
c06fff6e
ED		 3666 if (len > lv)
3667 len = lv;
8ae55f04
KK		 3668 if (put_user(len, optlen))
3669 return -EFAULT;
8dc41944
HX		 3670 if (copy_to_user(optval, data, len))
3671 return -EFAULT;
8ae55f04 3672 return 0;
1da177e4
LT		 3673}
3674
3675
351638e7
JP		 3676static int packet_notifier(struct notifier_block *this,
3677 unsigned long msg, void *ptr)
1da177e4
LT		 3678{
3679 struct sock *sk;
351638e7 3680 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3681 struct net *net = dev_net(dev);
1da177e4 3682
808f5114 3683 rcu_read_lock();
b67bfe0d 3684 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3685 struct packet_sock *po = pkt_sk(sk);
3686
3687 switch (msg) {
3688 case NETDEV_UNREGISTER:
1da177e4 3689 if (po->mclist)
82f17091 3690 packet_dev_mclist_delete(dev, &po->mclist);
a2efcfa0
DM		 3691 /* fallthrough */
3692
1da177e4
LT		 3693 case NETDEV_DOWN:
3694 if (dev->ifindex == po->ifindex) {
3695 spin_lock(&po->bind_lock);
3696 if (po->running) {
ce06b03e 3697 __unregister_prot_hook(sk, false);
1da177e4
LT		 3698 sk->sk_err = ENETDOWN;
3699 if (!sock_flag(sk, SOCK_DEAD))
3700 sk->sk_error_report(sk);
3701 }
3702 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3703 packet_cached_dev_reset(po);
1da177e4 3704 po->ifindex = -1;
160ff18a
BG		 3705 if (po->prot_hook.dev)
3706 dev_put(po->prot_hook.dev);
1da177e4
LT		 3707 po->prot_hook.dev = NULL;
3708 }
3709 spin_unlock(&po->bind_lock);
3710 }
3711 break;
3712 case NETDEV_UP:
808f5114 3713 if (dev->ifindex == po->ifindex) {
3714 spin_lock(&po->bind_lock);
ce06b03e
DM		 3715 if (po->num)
3716 register_prot_hook(sk);
808f5114 3717 spin_unlock(&po->bind_lock);
1da177e4 3718 }
1da177e4
LT		 3719 break;
3720 }
3721 }
808f5114 3722 rcu_read_unlock();
1da177e4
LT		 3723 return NOTIFY_DONE;
3724}
3725
3726
3727static int packet_ioctl(struct socket *sock, unsigned int cmd,
3728 unsigned long arg)
3729{
3730 struct sock *sk = sock->sk;
3731
69e3c75f 3732 switch (cmd) {
40d4e3df
ED		 3733 case SIOCOUTQ:
3734 {
3735 int amount = sk_wmem_alloc_get(sk);
31e6d363 3736
40d4e3df
ED		 3737 return put_user(amount, (int __user *)arg);
3738 }
3739 case SIOCINQ:
3740 {
3741 struct sk_buff *skb;
3742 int amount = 0;
3743
3744 spin_lock_bh(&sk->sk_receive_queue.lock);
3745 skb = skb_peek(&sk->sk_receive_queue);
3746 if (skb)
3747 amount = skb->len;
3748 spin_unlock_bh(&sk->sk_receive_queue.lock);
3749 return put_user(amount, (int __user *)arg);
3750 }
3751 case SIOCGSTAMP:
3752 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3753 case SIOCGSTAMPNS:
3754 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3755
1da177e4 3756#ifdef CONFIG_INET
40d4e3df
ED		 3757 case SIOCADDRT:
3758 case SIOCDELRT:
3759 case SIOCDARP:
3760 case SIOCGARP:
3761 case SIOCSARP:
3762 case SIOCGIFADDR:
3763 case SIOCSIFADDR:
3764 case SIOCGIFBRDADDR:
3765 case SIOCSIFBRDADDR:
3766 case SIOCGIFNETMASK:
3767 case SIOCSIFNETMASK:
3768 case SIOCGIFDSTADDR:
3769 case SIOCSIFDSTADDR:
3770 case SIOCSIFFLAGS:
40d4e3df 3771 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3772#endif
3773
40d4e3df
ED		 3774 default:
3775 return -ENOIOCTLCMD;
1da177e4
LT		 3776 }
3777 return 0;
3778}
3779
40d4e3df 3780static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3781 poll_table *wait)
3782{
3783 struct sock *sk = sock->sk;
3784 struct packet_sock *po = pkt_sk(sk);
3785 unsigned int mask = datagram_poll(file, sock, wait);
3786
3787 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3788 if (po->rx_ring.pg_vec) {
f6fb8f10 3789 if (!packet_previous_rx_frame(po, &po->rx_ring,
3790 TP_STATUS_KERNEL))
1da177e4
LT		 3791 mask |= POLLIN | POLLRDNORM;
3792 }
2ccdbaa6 3793 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
54d7c01d 3794 po->pressure = 0;
1da177e4 3795 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3796 spin_lock_bh(&sk->sk_write_queue.lock);
3797 if (po->tx_ring.pg_vec) {
3798 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3799 mask |= POLLOUT | POLLWRNORM;
3800 }
3801 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3802 return mask;
3803}
3804
3805
3806/* Dirty? Well, I still did not learn better way to account
3807 * for user mmaps.
3808 */
3809
3810static void packet_mm_open(struct vm_area_struct *vma)
3811{
3812 struct file *file = vma->vm_file;
40d4e3df 3813 struct socket *sock = file->private_data;
1da177e4 3814 struct sock *sk = sock->sk;
1ce4f28b 3815
1da177e4
LT		 3816 if (sk)
3817 atomic_inc(&pkt_sk(sk)->mapped);
3818}
3819
3820static void packet_mm_close(struct vm_area_struct *vma)
3821{
3822 struct file *file = vma->vm_file;
40d4e3df 3823 struct socket *sock = file->private_data;
1da177e4 3824 struct sock *sk = sock->sk;
1ce4f28b 3825
1da177e4
LT		 3826 if (sk)
3827 atomic_dec(&pkt_sk(sk)->mapped);
3828}
3829
f0f37e2f 3830static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3831 .open = packet_mm_open,
3832 .close = packet_mm_close,
1da177e4
LT		 3833};
3834
0e3125c7
NH		 3835static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3836 unsigned int len)
1da177e4
LT		 3837{
3838 int i;
3839
4ebf0ae2 3840 for (i = 0; i < len; i++) {
0e3125c7 3841 if (likely(pg_vec[i].buffer)) {
c56b4d90 3842 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3843 vfree(pg_vec[i].buffer);
3844 else
3845 free_pages((unsigned long)pg_vec[i].buffer,
3846 order);
3847 pg_vec[i].buffer = NULL;
3848 }
1da177e4
LT		 3849 }
3850 kfree(pg_vec);
3851}
3852
eea49cc9 3853static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3854{
f0d4eb29 3855 char *buffer;
0e3125c7
NH		 3856 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3857 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3858
3859 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3860 if (buffer)
3861 return buffer;
3862
f0d4eb29 3863 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3864 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3865 if (buffer)
3866 return buffer;
3867
f0d4eb29 3868 /* vmalloc failed, lets dig into swap here */
0e3125c7 3869 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3870 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3871 if (buffer)
3872 return buffer;
3873
f0d4eb29 3874 /* complete and utter failure */
0e3125c7 3875 return NULL;
4ebf0ae2
DM		 3876}
3877
0e3125c7 3878static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3879{
3880 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3881 struct pgv *pg_vec;
4ebf0ae2
DM		 3882 int i;
3883
0e3125c7 3884 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3885 if (unlikely(!pg_vec))
3886 goto out;
3887
3888 for (i = 0; i < block_nr; i++) {
c56b4d90 3889 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3890 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3891 goto out_free_pgvec;
3892 }
3893
3894out:
3895 return pg_vec;
3896
3897out_free_pgvec:
3898 free_pg_vec(pg_vec, order, block_nr);
3899 pg_vec = NULL;
3900 goto out;
3901}
1da177e4 3902
f6fb8f10 3903static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3904 int closing, int tx_ring)
1da177e4 3905{
0e3125c7 3906 struct pgv *pg_vec = NULL;
1da177e4 3907 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3908 int was_running, order = 0;
69e3c75f
JB		 3909 struct packet_ring_buffer *rb;
3910 struct sk_buff_head *rb_queue;
0e11c91e 3911 __be16 num;
f6fb8f10 3912 int err = -EINVAL;
3913 /* Added to avoid minimal code churn */
3914 struct tpacket_req *req = &req_u->req;
3915
3916 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3917 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3918 WARN(1, "Tx-ring is not supported.\n");
3919 goto out;
3920 }
1ce4f28b 3921
69e3c75f
JB		 3922 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3923 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3924
69e3c75f
JB		 3925 err = -EBUSY;
3926 if (!closing) {
3927 if (atomic_read(&po->mapped))
3928 goto out;
b0138408 3929 if (packet_read_pending(rb))
69e3c75f
JB		 3930 goto out;
3931 }
1da177e4 3932
69e3c75f
JB		 3933 if (req->tp_block_nr) {
3934 /* Sanity tests and some calculations */
3935 err = -EBUSY;
3936 if (unlikely(rb->pg_vec))
3937 goto out;
1da177e4 3938
bbd6ef87
PM		 3939 switch (po->tp_version) {
3940 case TPACKET_V1:
3941 po->tp_hdrlen = TPACKET_HDRLEN;
3942 break;
3943 case TPACKET_V2:
3944 po->tp_hdrlen = TPACKET2_HDRLEN;
3945 break;
f6fb8f10 3946 case TPACKET_V3:
3947 po->tp_hdrlen = TPACKET3_HDRLEN;
3948 break;
bbd6ef87
PM		 3949 }
3950
69e3c75f 3951 err = -EINVAL;
4ebf0ae2 3952 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3953 goto out;
4ebf0ae2 3954 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3955 goto out;
dc808110
ED		 3956 if (po->tp_version >= TPACKET_V3 &&
3957 (int)(req->tp_block_size -
3958 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3959 goto out;
8913336a 3960 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3961 po->tp_reserve))
3962 goto out;
4ebf0ae2 3963 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3964 goto out;
1da177e4 3965
69e3c75f
JB		 3966 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3967 if (unlikely(rb->frames_per_block <= 0))
3968 goto out;
3969 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3970 req->tp_frame_nr))
3971 goto out;
1da177e4
LT		 3972
3973 err = -ENOMEM;
4ebf0ae2
DM		 3974 order = get_order(req->tp_block_size);
3975 pg_vec = alloc_pg_vec(req, order);
3976 if (unlikely(!pg_vec))
1da177e4 3977 goto out;
f6fb8f10 3978 switch (po->tp_version) {
3979 case TPACKET_V3:
3980 /* Transmit path is not supported. We checked
3981 * it above but just being paranoid
3982 */
3983 if (!tx_ring)
e8e85cc5 3984 init_prb_bdqc(po, rb, pg_vec, req_u);
d7cf0c34 3985 break;
f6fb8f10 3986 default:
3987 break;
3988 }
69e3c75f
JB		 3989 }
3990 /* Done */
3991 else {
3992 err = -EINVAL;
4ebf0ae2 3993 if (unlikely(req->tp_frame_nr))
69e3c75f 3994 goto out;
1da177e4
LT		 3995 }
3996
3997 lock_sock(sk);
3998
3999 /* Detach socket from network */
4000 spin_lock(&po->bind_lock);
4001 was_running = po->running;
4002 num = po->num;
4003 if (was_running) {
1da177e4 4004 po->num = 0;
ce06b03e 4005 __unregister_prot_hook(sk, false);
1da177e4
LT		 4006 }
4007 spin_unlock(&po->bind_lock);
1ce4f28b 4008
1da177e4
LT		 4009 synchronize_net();
4010
4011 err = -EBUSY;
905db440 4012 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 4013 if (closing || atomic_read(&po->mapped) == 0) {
4014 err = 0;
69e3c75f 4015 spin_lock_bh(&rb_queue->lock);
c053fd96 4016 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 4017 rb->frame_max = (req->tp_frame_nr - 1);
4018 rb->head = 0;
4019 rb->frame_size = req->tp_frame_size;
4020 spin_unlock_bh(&rb_queue->lock);
4021
c053fd96
CG		 4022 swap(rb->pg_vec_order, order);
4023 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 4024
4025 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4026 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4027 tpacket_rcv : packet_rcv;
4028 skb_queue_purge(rb_queue);
1da177e4 4029 if (atomic_read(&po->mapped))
40d4e3df
ED		 4030 pr_err("packet_mmap: vma is busy: %d\n",
4031 atomic_read(&po->mapped));
1da177e4 4032 }
905db440 4033 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4034
4035 spin_lock(&po->bind_lock);
ce06b03e 4036 if (was_running) {
1da177e4 4037 po->num = num;
ce06b03e 4038 register_prot_hook(sk);
1da177e4
LT		 4039 }
4040 spin_unlock(&po->bind_lock);
f6fb8f10 4041 if (closing && (po->tp_version > TPACKET_V2)) {
4042 /* Because we don't support block-based V3 on tx-ring */
4043 if (!tx_ring)
73d0fcf2 4044 prb_shutdown_retire_blk_timer(po, rb_queue);
f6fb8f10 4045 }
1da177e4
LT		 4046 release_sock(sk);
4047
1da177e4
LT		 4048 if (pg_vec)
4049 free_pg_vec(pg_vec, order, req->tp_block_nr);
4050out:
4051 return err;
4052}
4053
69e3c75f
JB		 4054static int packet_mmap(struct file *file, struct socket *sock,
4055 struct vm_area_struct *vma)
1da177e4
LT		 4056{
4057 struct sock *sk = sock->sk;
4058 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 4059 unsigned long size, expected_size;
4060 struct packet_ring_buffer *rb;
1da177e4
LT		 4061 unsigned long start;
4062 int err = -EINVAL;
4063 int i;
4064
4065 if (vma->vm_pgoff)
4066 return -EINVAL;
4067
905db440 4068 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 4069
4070 expected_size = 0;
4071 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4072 if (rb->pg_vec) {
4073 expected_size += rb->pg_vec_len
4074 * rb->pg_vec_pages
4075 * PAGE_SIZE;
4076 }
4077 }
4078
4079 if (expected_size == 0)
1da177e4 4080 goto out;
69e3c75f
JB		 4081
4082 size = vma->vm_end - vma->vm_start;
4083 if (size != expected_size)
1da177e4
LT		 4084 goto out;
4085
1da177e4 4086 start = vma->vm_start;
69e3c75f
JB		 4087 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4088 if (rb->pg_vec == NULL)
4089 continue;
4090
4091 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 4092 struct page *page;
4093 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 4094 int pg_num;
4095
c56b4d90
CG		 4096 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4097 page = pgv_to_page(kaddr);
69e3c75f
JB		 4098 err = vm_insert_page(vma, start, page);
4099 if (unlikely(err))
4100 goto out;
4101 start += PAGE_SIZE;
0e3125c7 4102 kaddr += PAGE_SIZE;
69e3c75f 4103 }
4ebf0ae2 4104 }
1da177e4 4105 }
69e3c75f 4106
4ebf0ae2 4107 atomic_inc(&po->mapped);
1da177e4
LT		 4108 vma->vm_ops = &packet_mmap_ops;
4109 err = 0;
4110
4111out:
905db440 4112 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4113 return err;
4114}
1da177e4 4115
90ddc4f0 4116static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 4117 .family = PF_PACKET,
4118 .owner = THIS_MODULE,
4119 .release = packet_release,
4120 .bind = packet_bind_spkt,
4121 .connect = sock_no_connect,
4122 .socketpair = sock_no_socketpair,
4123 .accept = sock_no_accept,
4124 .getname = packet_getname_spkt,
4125 .poll = datagram_poll,
4126 .ioctl = packet_ioctl,
4127 .listen = sock_no_listen,
4128 .shutdown = sock_no_shutdown,
4129 .setsockopt = sock_no_setsockopt,
4130 .getsockopt = sock_no_getsockopt,
4131 .sendmsg = packet_sendmsg_spkt,
4132 .recvmsg = packet_recvmsg,
4133 .mmap = sock_no_mmap,
4134 .sendpage = sock_no_sendpage,
4135};
1da177e4 4136
90ddc4f0 4137static const struct proto_ops packet_ops = {
1da177e4
LT		 4138 .family = PF_PACKET,
4139 .owner = THIS_MODULE,
4140 .release = packet_release,
4141 .bind = packet_bind,
4142 .connect = sock_no_connect,
4143 .socketpair = sock_no_socketpair,
4144 .accept = sock_no_accept,
1ce4f28b 4145 .getname = packet_getname,
1da177e4
LT		 4146 .poll = packet_poll,
4147 .ioctl = packet_ioctl,
4148 .listen = sock_no_listen,
4149 .shutdown = sock_no_shutdown,
4150 .setsockopt = packet_setsockopt,
4151 .getsockopt = packet_getsockopt,
4152 .sendmsg = packet_sendmsg,
4153 .recvmsg = packet_recvmsg,
4154 .mmap = packet_mmap,
4155 .sendpage = sock_no_sendpage,
4156};
4157
ec1b4cf7 4158static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4159 .family = PF_PACKET,
4160 .create = packet_create,
4161 .owner = THIS_MODULE,
4162};
4163
4164static struct notifier_block packet_netdev_notifier = {
40d4e3df 4165 .notifier_call = packet_notifier,
1da177e4
LT		 4166};
4167
4168#ifdef CONFIG_PROC_FS
1da177e4
LT		 4169
4170static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4171 __acquires(RCU)
1da177e4 4172{
e372c414 4173 struct net *net = seq_file_net(seq);
808f5114 4174
4175 rcu_read_lock();
4176 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4177}
4178
4179static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4180{
1bf40954 4181 struct net *net = seq_file_net(seq);
808f5114 4182 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4183}
4184
4185static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4186 __releases(RCU)
1da177e4 4187{
808f5114 4188 rcu_read_unlock();
1da177e4
LT		 4189}
4190
1ce4f28b 4191static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4192{
4193 if (v == SEQ_START_TOKEN)
4194 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4195 else {
b7ceabd9 4196 struct sock *s = sk_entry(v);
1da177e4
LT		 4197 const struct packet_sock *po = pkt_sk(s);
4198
4199 seq_printf(seq,
71338aa7 4200 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4201 s,
4202 atomic_read(&s->sk_refcnt),
4203 s->sk_type,
4204 ntohs(po->num),
4205 po->ifindex,
4206 po->running,
4207 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4208 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4209 sock_i_ino(s));
1da177e4
LT		 4210 }
4211
4212 return 0;
4213}
4214
56b3d975 4215static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4216 .start = packet_seq_start,
4217 .next = packet_seq_next,
4218 .stop = packet_seq_stop,
4219 .show = packet_seq_show,
4220};
4221
4222static int packet_seq_open(struct inode *inode, struct file *file)
4223{
e372c414
DL		 4224 return seq_open_net(inode, file, &packet_seq_ops,
4225 sizeof(struct seq_net_private));
1da177e4
LT		 4226}
4227
da7071d7 4228static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4229 .owner = THIS_MODULE,
4230 .open = packet_seq_open,
4231 .read = seq_read,
4232 .llseek = seq_lseek,
e372c414 4233 .release = seq_release_net,
1da177e4
LT		 4234};
4235
4236#endif
4237
2c8c1e72 4238static int __net_init packet_net_init(struct net *net)
d12d01d6 4239{
0fa7fa98 4240 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4241 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4242
d4beaa66 4243 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4244 return -ENOMEM;
4245
4246 return 0;
4247}
4248
2c8c1e72 4249static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4250{
ece31ffd 4251 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4252}
4253
4254static struct pernet_operations packet_net_ops = {
4255 .init = packet_net_init,
4256 .exit = packet_net_exit,
4257};
4258
4259
1da177e4
LT		 4260static void __exit packet_exit(void)
4261{
1da177e4 4262 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4263 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4264 sock_unregister(PF_PACKET);
4265 proto_unregister(&packet_proto);
4266}
4267
4268static int __init packet_init(void)
4269{
4270 int rc = proto_register(&packet_proto, 0);
4271
4272 if (rc != 0)
4273 goto out;
4274
4275 sock_register(&packet_family_ops);
d12d01d6 4276 register_pernet_subsys(&packet_net_ops);
1da177e4 4277 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4278out:
4279 return rc;
4280}
4281
4282module_init(packet_init);
4283module_exit(packet_exit);
4284MODULE_LICENSE("GPL");
4285MODULE_ALIAS_NETPROTO(PF_PACKET);
ubuntu-zesty-kernel mirror