]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - net/packet/af_packet.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
packet: rollover prepare: per-socket state
[mirror_ubuntu-hirsute-kernel.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111 218struct packet_skb_cb {
ffbc6111
HX		 219 union {
220 struct sockaddr_pkt pkt;
2472d761
EB		 221 union {
222 /* Trick: alias skb original length with
223 * ll.sll_family and ll.protocol in order
224 * to save room.
225 */
226 unsigned int origlen;
227 struct sockaddr_ll ll;
228 };
ffbc6111
HX		 229 } sa;
230};
231
232#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 233
bc59ba39 234#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 235#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 236 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 237#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 238 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 239#define GET_NEXT_PRB_BLK_NUM(x) \
240 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
241 ((x)->kactive_blk_num+1) : 0)
242
dc99f600
DM		 243static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244static void __fanout_link(struct sock *sk, struct packet_sock *po);
245
d346a3fa
DB		 246static int packet_direct_xmit(struct sk_buff *skb)
247{
248 struct net_device *dev = skb->dev;
d346a3fa
DB		 249 netdev_features_t features;
250 struct netdev_queue *txq;
43279500 251 int ret = NETDEV_TX_BUSY;
d346a3fa
DB		 252
253 if (unlikely(!netif_running(dev) ||
43279500
DB		 254 !netif_carrier_ok(dev)))
255 goto drop;
d346a3fa
DB		 256
257 features = netif_skb_features(skb);
258 if (skb_needs_linearize(skb, features) &&
43279500
DB		 259 __skb_linearize(skb))
260 goto drop;
d346a3fa 261
10c51b56 262 txq = skb_get_tx_queue(dev, skb);
d346a3fa 263
43279500
DB		 264 local_bh_disable();
265
266 HARD_TX_LOCK(dev, txq, smp_processor_id());
10b3ad8c 267 if (!netif_xmit_frozen_or_drv_stopped(txq))
fa2dbdc2 268 ret = netdev_start_xmit(skb, dev, txq, false);
43279500 269 HARD_TX_UNLOCK(dev, txq);
d346a3fa 270
43279500
DB		 271 local_bh_enable();
272
273 if (!dev_xmit_complete(ret))
d346a3fa 274 kfree_skb(skb);
43279500 275
d346a3fa 276 return ret;
43279500 277drop:
0f97ede4 278 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 279 kfree_skb(skb);
280 return NET_XMIT_DROP;
d346a3fa
DB		 281}
282
66e56cd4
DB		 283static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284{
285 struct net_device *dev;
286
287 rcu_read_lock();
288 dev = rcu_dereference(po->cached_dev);
289 if (likely(dev))
290 dev_hold(dev);
291 rcu_read_unlock();
292
293 return dev;
294}
295
296static void packet_cached_dev_assign(struct packet_sock *po,
297 struct net_device *dev)
298{
299 rcu_assign_pointer(po->cached_dev, dev);
300}
301
302static void packet_cached_dev_reset(struct packet_sock *po)
303{
304 RCU_INIT_POINTER(po->cached_dev, NULL);
305}
306
d346a3fa
DB		 307static bool packet_use_direct_xmit(const struct packet_sock *po)
308{
309 return po->xmit == packet_direct_xmit;
310}
311
0fd5d57b 312static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 313{
1cbac010 314 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 315}
316
0fd5d57b
DB		 317static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
318{
319 const struct net_device_ops *ops = dev->netdev_ops;
320 u16 queue_index;
321
322 if (ops->ndo_select_queue) {
323 queue_index = ops->ndo_select_queue(dev, skb, NULL,
324 __packet_pick_tx_queue);
325 queue_index = netdev_cap_txqueue(dev, queue_index);
326 } else {
327 queue_index = __packet_pick_tx_queue(dev, skb);
328 }
329
330 skb_set_queue_mapping(skb, queue_index);
331}
332
ce06b03e
DM		 333/* register_prot_hook must be invoked with the po->bind_lock held,
334 * or from a context in which asynchronous accesses to the packet
335 * socket is not possible (packet_create()).
336 */
337static void register_prot_hook(struct sock *sk)
338{
339 struct packet_sock *po = pkt_sk(sk);
e40526cb 340
ce06b03e 341 if (!po->running) {
66e56cd4 342 if (po->fanout)
dc99f600 343 __fanout_link(sk, po);
66e56cd4 344 else
dc99f600 345 dev_add_pack(&po->prot_hook);
e40526cb 346
ce06b03e
DM		 347 sock_hold(sk);
348 po->running = 1;
349 }
350}
351
352/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
353 * held. If the sync parameter is true, we will temporarily drop
354 * the po->bind_lock and do a synchronize_net to make sure no
355 * asynchronous packet processing paths still refer to the elements
356 * of po->prot_hook. If the sync parameter is false, it is the
357 * callers responsibility to take care of this.
358 */
359static void __unregister_prot_hook(struct sock *sk, bool sync)
360{
361 struct packet_sock *po = pkt_sk(sk);
362
363 po->running = 0;
66e56cd4
DB		 364
365 if (po->fanout)
dc99f600 366 __fanout_unlink(sk, po);
66e56cd4 367 else
dc99f600 368 __dev_remove_pack(&po->prot_hook);
e40526cb 369
ce06b03e
DM		 370 __sock_put(sk);
371
372 if (sync) {
373 spin_unlock(&po->bind_lock);
374 synchronize_net();
375 spin_lock(&po->bind_lock);
376 }
377}
378
379static void unregister_prot_hook(struct sock *sk, bool sync)
380{
381 struct packet_sock *po = pkt_sk(sk);
382
383 if (po->running)
384 __unregister_prot_hook(sk, sync);
385}
386
6e58040b 387static inline struct page * __pure pgv_to_page(void *addr)
0af55bb5
CG		 388{
389 if (is_vmalloc_addr(addr))
390 return vmalloc_to_page(addr);
391 return virt_to_page(addr);
392}
393
69e3c75f 394static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 395{
184f489e 396 union tpacket_uhdr h;
1da177e4 397
69e3c75f 398 h.raw = frame;
bbd6ef87
PM		 399 switch (po->tp_version) {
400 case TPACKET_V1:
69e3c75f 401 h.h1->tp_status = status;
0af55bb5 402 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 403 break;
404 case TPACKET_V2:
69e3c75f 405 h.h2->tp_status = status;
0af55bb5 406 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 407 break;
f6fb8f10 408 case TPACKET_V3:
69e3c75f 409 default:
f6fb8f10 410 WARN(1, "TPACKET version not supported.\n");
69e3c75f 411 BUG();
bbd6ef87 412 }
69e3c75f
JB		 413
414 smp_wmb();
bbd6ef87
PM		 415}
416
69e3c75f 417static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 418{
184f489e 419 union tpacket_uhdr h;
bbd6ef87 420
69e3c75f
JB		 421 smp_rmb();
422
bbd6ef87
PM		 423 h.raw = frame;
424 switch (po->tp_version) {
425 case TPACKET_V1:
0af55bb5 426 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 427 return h.h1->tp_status;
bbd6ef87 428 case TPACKET_V2:
0af55bb5 429 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 430 return h.h2->tp_status;
f6fb8f10 431 case TPACKET_V3:
69e3c75f 432 default:
f6fb8f10 433 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 434 BUG();
435 return 0;
bbd6ef87 436 }
1da177e4 437}
69e3c75f 438
b9c32fb2
DB		 439static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
440 unsigned int flags)
7a51384c
DB		 441{
442 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
443
68a360e8
WB		 444 if (shhwtstamps &&
445 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
446 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
447 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 448
449 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 450 return TP_STATUS_TS_SOFTWARE;
7a51384c 451
b9c32fb2 452 return 0;
7a51384c
DB		 453}
454
b9c32fb2
DB		 455static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
456 struct sk_buff *skb)
2e31396f
WB		 457{
458 union tpacket_uhdr h;
459 struct timespec ts;
b9c32fb2 460 __u32 ts_status;
2e31396f 461
b9c32fb2
DB		 462 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
463 return 0;
2e31396f
WB		 464
465 h.raw = frame;
466 switch (po->tp_version) {
467 case TPACKET_V1:
468 h.h1->tp_sec = ts.tv_sec;
469 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
470 break;
471 case TPACKET_V2:
472 h.h2->tp_sec = ts.tv_sec;
473 h.h2->tp_nsec = ts.tv_nsec;
474 break;
475 case TPACKET_V3:
476 default:
477 WARN(1, "TPACKET version not supported.\n");
478 BUG();
479 }
480
481 /* one flush is safe, as both fields always lie on the same cacheline */
482 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
483 smp_wmb();
b9c32fb2
DB		 484
485 return ts_status;
2e31396f
WB		 486}
487
69e3c75f
JB		 488static void *packet_lookup_frame(struct packet_sock *po,
489 struct packet_ring_buffer *rb,
490 unsigned int position,
491 int status)
492{
493 unsigned int pg_vec_pos, frame_offset;
184f489e 494 union tpacket_uhdr h;
69e3c75f
JB		 495
496 pg_vec_pos = position / rb->frames_per_block;
497 frame_offset = position % rb->frames_per_block;
498
0e3125c7
NH		 499 h.raw = rb->pg_vec[pg_vec_pos].buffer +
500 (frame_offset * rb->frame_size);
69e3c75f
JB		 501
502 if (status != __packet_get_status(po, h.raw))
503 return NULL;
504
505 return h.raw;
506}
507
eea49cc9 508static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 509 struct packet_ring_buffer *rb,
510 int status)
511{
512 return packet_lookup_frame(po, rb, rb->head, status);
513}
514
bc59ba39 515static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 516{
517 del_timer_sync(&pkc->retire_blk_timer);
518}
519
520static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
521 int tx_ring,
522 struct sk_buff_head *rb_queue)
523{
bc59ba39 524 struct tpacket_kbdq_core *pkc;
f6fb8f10 525
22781a5b
DJ		 526 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
527 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 528
ec6f809f 529 spin_lock_bh(&rb_queue->lock);
f6fb8f10 530 pkc->delete_blk_timer = 1;
ec6f809f 531 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 532
533 prb_del_retire_blk_timer(pkc);
534}
535
536static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 537 struct tpacket_kbdq_core *pkc,
f6fb8f10 538 void (*func) (unsigned long))
539{
540 init_timer(&pkc->retire_blk_timer);
541 pkc->retire_blk_timer.data = (long)po;
542 pkc->retire_blk_timer.function = func;
543 pkc->retire_blk_timer.expires = jiffies;
544}
545
546static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
547{
bc59ba39 548 struct tpacket_kbdq_core *pkc;
f6fb8f10 549
550 if (tx_ring)
551 BUG();
552
22781a5b
DJ		 553 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
554 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 555 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
556}
557
558static int prb_calc_retire_blk_tmo(struct packet_sock *po,
559 int blk_size_in_bytes)
560{
561 struct net_device *dev;
562 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 563 struct ethtool_cmd ecmd;
564 int err;
e440cf2c 565 u32 speed;
f6fb8f10 566
4bc71cb9
JP		 567 rtnl_lock();
568 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
569 if (unlikely(!dev)) {
570 rtnl_unlock();
f6fb8f10 571 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 572 }
573 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 574 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 575 rtnl_unlock();
576 if (!err) {
4bc71cb9
JP		 577 /*
578 * If the link speed is so slow you don't really
579 * need to worry about perf anyways
580 */
e440cf2c 581 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 582 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 583 } else {
584 msec = 1;
585 div = speed / 1000;
f6fb8f10 586 }
587 }
588
589 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
590
591 if (div)
592 mbits /= div;
593
594 tmo = mbits * msec;
595
596 if (div)
597 return tmo+1;
598 return tmo;
599}
600
bc59ba39 601static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 602 union tpacket_req_u *req_u)
603{
604 p1->feature_req_word = req_u->req3.tp_feature_req_word;
605}
606
607static void init_prb_bdqc(struct packet_sock *po,
608 struct packet_ring_buffer *rb,
609 struct pgv *pg_vec,
610 union tpacket_req_u *req_u, int tx_ring)
611{
22781a5b 612 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 613 struct tpacket_block_desc *pbd;
f6fb8f10 614
615 memset(p1, 0x0, sizeof(*p1));
616
617 p1->knxt_seq_num = 1;
618 p1->pkbdq = pg_vec;
bc59ba39 619 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 620 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 621 p1->kblk_size = req_u->req3.tp_block_size;
622 p1->knum_blocks = req_u->req3.tp_block_nr;
623 p1->hdrlen = po->tp_hdrlen;
624 p1->version = po->tp_version;
625 p1->last_kactive_blk_num = 0;
ee80fbf3 626 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 627 if (req_u->req3.tp_retire_blk_tov)
628 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
629 else
630 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
631 req_u->req3.tp_block_size);
632 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
633 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
634
dc808110 635 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 636 prb_init_ft_ops(p1, req_u);
637 prb_setup_retire_blk_timer(po, tx_ring);
638 prb_open_block(p1, pbd);
639}
640
641/* Do NOT update the last_blk_num first.
642 * Assumes sk_buff_head lock is held.
643 */
bc59ba39 644static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 645{
646 mod_timer(&pkc->retire_blk_timer,
647 jiffies + pkc->tov_in_jiffies);
648 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
649}
650
651/*
652 * Timer logic:
653 * 1) We refresh the timer only when we open a block.
654 * By doing this we don't waste cycles refreshing the timer
655 * on packet-by-packet basis.
656 *
657 * With a 1MB block-size, on a 1Gbps line, it will take
658 * i) ~8 ms to fill a block + ii) memcpy etc.
659 * In this cut we are not accounting for the memcpy time.
660 *
661 * So, if the user sets the 'tmo' to 10ms then the timer
662 * will never fire while the block is still getting filled
663 * (which is what we want). However, the user could choose
664 * to close a block early and that's fine.
665 *
666 * But when the timer does fire, we check whether or not to refresh it.
667 * Since the tmo granularity is in msecs, it is not too expensive
668 * to refresh the timer, lets say every '8' msecs.
669 * Either the user can set the 'tmo' or we can derive it based on
670 * a) line-speed and b) block-size.
671 * prb_calc_retire_blk_tmo() calculates the tmo.
672 *
673 */
674static void prb_retire_rx_blk_timer_expired(unsigned long data)
675{
676 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 677 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 678 unsigned int frozen;
bc59ba39 679 struct tpacket_block_desc *pbd;
f6fb8f10 680
681 spin_lock(&po->sk.sk_receive_queue.lock);
682
683 frozen = prb_queue_frozen(pkc);
684 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
685
686 if (unlikely(pkc->delete_blk_timer))
687 goto out;
688
689 /* We only need to plug the race when the block is partially filled.
690 * tpacket_rcv:
691 * lock(); increment BLOCK_NUM_PKTS; unlock()
692 * copy_bits() is in progress ...
693 * timer fires on other cpu:
694 * we can't retire the current block because copy_bits
695 * is in progress.
696 *
697 */
698 if (BLOCK_NUM_PKTS(pbd)) {
699 while (atomic_read(&pkc->blk_fill_in_prog)) {
700 /* Waiting for skb_copy_bits to finish... */
701 cpu_relax();
702 }
703 }
704
705 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
706 if (!frozen) {
41a50d62
AD		 707 if (!BLOCK_NUM_PKTS(pbd)) {
708 /* An empty block. Just refresh the timer. */
709 goto refresh_timer;
710 }
f6fb8f10 711 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
712 if (!prb_dispatch_next_block(pkc, po))
713 goto refresh_timer;
714 else
715 goto out;
716 } else {
717 /* Case 1. Queue was frozen because user-space was
718 * lagging behind.
719 */
720 if (prb_curr_blk_in_use(pkc, pbd)) {
721 /*
722 * Ok, user-space is still behind.
723 * So just refresh the timer.
724 */
725 goto refresh_timer;
726 } else {
727 /* Case 2. queue was frozen,user-space caught up,
728 * now the link went idle && the timer fired.
729 * We don't have a block to close.So we open this
730 * block and restart the timer.
731 * opening a block thaws the queue,restarts timer
732 * Thawing/timer-refresh is a side effect.
733 */
734 prb_open_block(pkc, pbd);
735 goto out;
736 }
737 }
738 }
739
740refresh_timer:
741 _prb_refresh_rx_retire_blk_timer(pkc);
742
743out:
744 spin_unlock(&po->sk.sk_receive_queue.lock);
745}
746
eea49cc9 747static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 748 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 749{
750 /* Flush everything minus the block header */
751
752#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
753 u8 *start, *end;
754
755 start = (u8 *)pbd1;
756
757 /* Skip the block header(we know header WILL fit in 4K) */
758 start += PAGE_SIZE;
759
760 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
761 for (; start < end; start += PAGE_SIZE)
762 flush_dcache_page(pgv_to_page(start));
763
764 smp_wmb();
765#endif
766
767 /* Now update the block status. */
768
769 BLOCK_STATUS(pbd1) = status;
770
771 /* Flush the block header */
772
773#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
774 start = (u8 *)pbd1;
775 flush_dcache_page(pgv_to_page(start));
776
777 smp_wmb();
778#endif
779}
780
781/*
782 * Side effect:
783 *
784 * 1) flush the block
785 * 2) Increment active_blk_num
786 *
787 * Note:We DONT refresh the timer on purpose.
788 * Because almost always the next block will be opened.
789 */
bc59ba39 790static void prb_close_block(struct tpacket_kbdq_core *pkc1,
791 struct tpacket_block_desc *pbd1,
f6fb8f10 792 struct packet_sock *po, unsigned int stat)
793{
794 __u32 status = TP_STATUS_USER | stat;
795
796 struct tpacket3_hdr *last_pkt;
bc59ba39 797 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
da413eec 798 struct sock *sk = &po->sk;
f6fb8f10 799
ee80fbf3 800 if (po->stats.stats3.tp_drops)
f6fb8f10 801 status |= TP_STATUS_LOSING;
802
803 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
804 last_pkt->tp_next_offset = 0;
805
806 /* Get the ts of the last pkt */
807 if (BLOCK_NUM_PKTS(pbd1)) {
808 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
809 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
810 } else {
41a50d62
AD		 811 /* Ok, we tmo'd - so get the current time.
812 *
813 * It shouldn't really happen as we don't close empty
814 * blocks. See prb_retire_rx_blk_timer_expired().
815 */
f6fb8f10 816 struct timespec ts;
817 getnstimeofday(&ts);
818 h1->ts_last_pkt.ts_sec = ts.tv_sec;
819 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
820 }
821
822 smp_wmb();
823
824 /* Flush the block */
825 prb_flush_block(pkc1, pbd1, status);
826
da413eec
DC		 827 sk->sk_data_ready(sk);
828
f6fb8f10 829 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
830}
831
eea49cc9 832static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 833{
834 pkc->reset_pending_on_curr_blk = 0;
835}
836
837/*
838 * Side effect of opening a block:
839 *
840 * 1) prb_queue is thawed.
841 * 2) retire_blk_timer is refreshed.
842 *
843 */
bc59ba39 844static void prb_open_block(struct tpacket_kbdq_core *pkc1,
845 struct tpacket_block_desc *pbd1)
f6fb8f10 846{
847 struct timespec ts;
bc59ba39 848 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 849
850 smp_rmb();
851
8da3056c
DB		 852 /* We could have just memset this but we will lose the
853 * flexibility of making the priv area sticky
854 */
f6fb8f10 855
8da3056c
DB		 856 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
857 BLOCK_NUM_PKTS(pbd1) = 0;
858 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 859
8da3056c
DB		 860 getnstimeofday(&ts);
861
862 h1->ts_first_pkt.ts_sec = ts.tv_sec;
863 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 864
8da3056c
DB		 865 pkc1->pkblk_start = (char *)pbd1;
866 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
867
868 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
869 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
870
871 pbd1->version = pkc1->version;
872 pkc1->prev = pkc1->nxt_offset;
873 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
874
875 prb_thaw_queue(pkc1);
876 _prb_refresh_rx_retire_blk_timer(pkc1);
877
878 smp_wmb();
f6fb8f10 879}
880
881/*
882 * Queue freeze logic:
883 * 1) Assume tp_block_nr = 8 blocks.
884 * 2) At time 't0', user opens Rx ring.
885 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
886 * 4) user-space is either sleeping or processing block '0'.
887 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
888 * it will close block-7,loop around and try to fill block '0'.
889 * call-flow:
890 * __packet_lookup_frame_in_block
891 * prb_retire_current_block()
892 * prb_dispatch_next_block()
893 * |->(BLOCK_STATUS == USER) evaluates to true
894 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
895 * 6) Now there are two cases:
896 * 6.1) Link goes idle right after the queue is frozen.
897 * But remember, the last open_block() refreshed the timer.
898 * When this timer expires,it will refresh itself so that we can
899 * re-open block-0 in near future.
900 * 6.2) Link is busy and keeps on receiving packets. This is a simple
901 * case and __packet_lookup_frame_in_block will check if block-0
902 * is free and can now be re-used.
903 */
eea49cc9 904static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 905 struct packet_sock *po)
906{
907 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 908 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 909}
910
911#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
912
913/*
914 * If the next block is free then we will dispatch it
915 * and return a good offset.
916 * Else, we will freeze the queue.
917 * So, caller must check the return value.
918 */
bc59ba39 919static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 920 struct packet_sock *po)
921{
bc59ba39 922 struct tpacket_block_desc *pbd;
f6fb8f10 923
924 smp_rmb();
925
926 /* 1. Get current block num */
927 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
928
929 /* 2. If this block is currently in_use then freeze the queue */
930 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
931 prb_freeze_queue(pkc, po);
932 return NULL;
933 }
934
935 /*
936 * 3.
937 * open this block and return the offset where the first packet
938 * needs to get stored.
939 */
940 prb_open_block(pkc, pbd);
941 return (void *)pkc->nxt_offset;
942}
943
bc59ba39 944static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 945 struct packet_sock *po, unsigned int status)
946{
bc59ba39 947 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 948
949 /* retire/close the current block */
950 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
951 /*
952 * Plug the case where copy_bits() is in progress on
953 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
954 * have space to copy the pkt in the current block and
955 * called prb_retire_current_block()
956 *
957 * We don't need to worry about the TMO case because
958 * the timer-handler already handled this case.
959 */
960 if (!(status & TP_STATUS_BLK_TMO)) {
961 while (atomic_read(&pkc->blk_fill_in_prog)) {
962 /* Waiting for skb_copy_bits to finish... */
963 cpu_relax();
964 }
965 }
966 prb_close_block(pkc, pbd, po, status);
967 return;
968 }
f6fb8f10 969}
970
eea49cc9 971static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 972 struct tpacket_block_desc *pbd)
f6fb8f10 973{
974 return TP_STATUS_USER & BLOCK_STATUS(pbd);
975}
976
eea49cc9 977static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 978{
979 return pkc->reset_pending_on_curr_blk;
980}
981
eea49cc9 982static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 983{
bc59ba39 984 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 985 atomic_dec(&pkc->blk_fill_in_prog);
986}
987
eea49cc9 988static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 989 struct tpacket3_hdr *ppd)
990{
3958afa1 991 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 992}
993
eea49cc9 994static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 995 struct tpacket3_hdr *ppd)
996{
997 ppd->hv1.tp_rxhash = 0;
998}
999
eea49cc9 1000static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 1001 struct tpacket3_hdr *ppd)
1002{
df8a39de
JP		 1003 if (skb_vlan_tag_present(pkc->skb)) {
1004 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
a0cdfcf3
AW		 1005 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1006 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 1007 } else {
9e67030a 1008 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1009 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1010 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1011 }
1012}
1013
bc59ba39 1014static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1015 struct tpacket3_hdr *ppd)
1016{
a0cdfcf3 1017 ppd->hv1.tp_padding = 0;
f6fb8f10 1018 prb_fill_vlan_info(pkc, ppd);
1019
1020 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1021 prb_fill_rxhash(pkc, ppd);
1022 else
1023 prb_clear_rxhash(pkc, ppd);
1024}
1025
eea49cc9 1026static void prb_fill_curr_block(char *curr,
bc59ba39 1027 struct tpacket_kbdq_core *pkc,
1028 struct tpacket_block_desc *pbd,
f6fb8f10 1029 unsigned int len)
1030{
1031 struct tpacket3_hdr *ppd;
1032
1033 ppd = (struct tpacket3_hdr *)curr;
1034 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1035 pkc->prev = curr;
1036 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1037 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1038 BLOCK_NUM_PKTS(pbd) += 1;
1039 atomic_inc(&pkc->blk_fill_in_prog);
1040 prb_run_all_ft_ops(pkc, ppd);
1041}
1042
1043/* Assumes caller has the sk->rx_queue.lock */
1044static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1045 struct sk_buff *skb,
1046 int status,
1047 unsigned int len
1048 )
1049{
bc59ba39 1050 struct tpacket_kbdq_core *pkc;
1051 struct tpacket_block_desc *pbd;
f6fb8f10 1052 char *curr, *end;
1053
e3192690 1054 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1055 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1056
1057 /* Queue is frozen when user space is lagging behind */
1058 if (prb_queue_frozen(pkc)) {
1059 /*
1060 * Check if that last block which caused the queue to freeze,
1061 * is still in_use by user-space.
1062 */
1063 if (prb_curr_blk_in_use(pkc, pbd)) {
1064 /* Can't record this packet */
1065 return NULL;
1066 } else {
1067 /*
1068 * Ok, the block was released by user-space.
1069 * Now let's open that block.
1070 * opening a block also thaws the queue.
1071 * Thawing is a side effect.
1072 */
1073 prb_open_block(pkc, pbd);
1074 }
1075 }
1076
1077 smp_mb();
1078 curr = pkc->nxt_offset;
1079 pkc->skb = skb;
e3192690 1080 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1081
1082 /* first try the current block */
1083 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1084 prb_fill_curr_block(curr, pkc, pbd, len);
1085 return (void *)curr;
1086 }
1087
1088 /* Ok, close the current block */
1089 prb_retire_current_block(pkc, po, 0);
1090
1091 /* Now, try to dispatch the next block */
1092 curr = (char *)prb_dispatch_next_block(pkc, po);
1093 if (curr) {
1094 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1095 prb_fill_curr_block(curr, pkc, pbd, len);
1096 return (void *)curr;
1097 }
1098
1099 /*
1100 * No free blocks are available.user_space hasn't caught up yet.
1101 * Queue was just frozen and now this packet will get dropped.
1102 */
1103 return NULL;
1104}
1105
eea49cc9 1106static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1107 struct sk_buff *skb,
1108 int status, unsigned int len)
1109{
1110 char *curr = NULL;
1111 switch (po->tp_version) {
1112 case TPACKET_V1:
1113 case TPACKET_V2:
1114 curr = packet_lookup_frame(po, &po->rx_ring,
1115 po->rx_ring.head, status);
1116 return curr;
1117 case TPACKET_V3:
1118 return __packet_lookup_frame_in_block(po, skb, status, len);
1119 default:
1120 WARN(1, "TPACKET version not supported\n");
1121 BUG();
99aa3473 1122 return NULL;
f6fb8f10 1123 }
1124}
1125
eea49cc9 1126static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1127 struct packet_ring_buffer *rb,
77f65ebd 1128 unsigned int idx,
f6fb8f10 1129 int status)
1130{
bc59ba39 1131 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1132 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1133
1134 if (status != BLOCK_STATUS(pbd))
1135 return NULL;
1136 return pbd;
1137}
1138
eea49cc9 1139static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1140{
1141 unsigned int prev;
1142 if (rb->prb_bdqc.kactive_blk_num)
1143 prev = rb->prb_bdqc.kactive_blk_num-1;
1144 else
1145 prev = rb->prb_bdqc.knum_blocks-1;
1146 return prev;
1147}
1148
1149/* Assumes caller has held the rx_queue.lock */
eea49cc9 1150static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1151 struct packet_ring_buffer *rb,
1152 int status)
1153{
1154 unsigned int previous = prb_previous_blk_num(rb);
1155 return prb_lookup_block(po, rb, previous, status);
1156}
1157
eea49cc9 1158static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1159 struct packet_ring_buffer *rb,
1160 int status)
1161{
1162 if (po->tp_version <= TPACKET_V2)
1163 return packet_previous_frame(po, rb, status);
1164
1165 return __prb_previous_block(po, rb, status);
1166}
1167
eea49cc9 1168static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1169 struct packet_ring_buffer *rb)
1170{
1171 switch (po->tp_version) {
1172 case TPACKET_V1:
1173 case TPACKET_V2:
1174 return packet_increment_head(rb);
1175 case TPACKET_V3:
1176 default:
1177 WARN(1, "TPACKET version not supported.\n");
1178 BUG();
1179 return;
1180 }
1181}
1182
eea49cc9 1183static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1184 struct packet_ring_buffer *rb,
1185 int status)
1186{
1187 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1188 return packet_lookup_frame(po, rb, previous, status);
1189}
1190
eea49cc9 1191static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1192{
1193 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1194}
1195
b0138408
DB		 1196static void packet_inc_pending(struct packet_ring_buffer *rb)
1197{
1198 this_cpu_inc(*rb->pending_refcnt);
1199}
1200
1201static void packet_dec_pending(struct packet_ring_buffer *rb)
1202{
1203 this_cpu_dec(*rb->pending_refcnt);
1204}
1205
1206static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1207{
1208 unsigned int refcnt = 0;
1209 int cpu;
1210
1211 /* We don't use pending refcount in rx_ring. */
1212 if (rb->pending_refcnt == NULL)
1213 return 0;
1214
1215 for_each_possible_cpu(cpu)
1216 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1217
1218 return refcnt;
1219}
1220
1221static int packet_alloc_pending(struct packet_sock *po)
1222{
1223 po->rx_ring.pending_refcnt = NULL;
1224
1225 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1226 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1227 return -ENOBUFS;
1228
1229 return 0;
1230}
1231
1232static void packet_free_pending(struct packet_sock *po)
1233{
1234 free_percpu(po->tx_ring.pending_refcnt);
1235}
1236
77f65ebd
WB		 1237static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1238{
1239 struct sock *sk = &po->sk;
1240 bool has_room;
1241
1242 if (po->prot_hook.func != tpacket_rcv)
1243 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1244 <= sk->sk_rcvbuf;
1245
1246 spin_lock(&sk->sk_receive_queue.lock);
1247 if (po->tp_version == TPACKET_V3)
1248 has_room = prb_lookup_block(po, &po->rx_ring,
1249 po->rx_ring.prb_bdqc.kactive_blk_num,
1250 TP_STATUS_KERNEL);
1251 else
1252 has_room = packet_lookup_frame(po, &po->rx_ring,
1253 po->rx_ring.head,
1254 TP_STATUS_KERNEL);
1255 spin_unlock(&sk->sk_receive_queue.lock);
1256
1257 return has_room;
1258}
1259
1da177e4
LT		 1260static void packet_sock_destruct(struct sock *sk)
1261{
ed85b565
RC		 1262 skb_queue_purge(&sk->sk_error_queue);
1263
547b792c
IJ		 1264 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1265 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1266
1267 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1268 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1269 return;
1270 }
1271
17ab56a2 1272 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1273}
1274
dc99f600
DM		 1275static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1276{
1277 int x = atomic_read(&f->rr_cur) + 1;
1278
1279 if (x >= num)
1280 x = 0;
1281
1282 return x;
1283}
1284
77f65ebd
WB		 1285static unsigned int fanout_demux_hash(struct packet_fanout *f,
1286 struct sk_buff *skb,
1287 unsigned int num)
dc99f600 1288{
61b905da 1289 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1290}
1291
77f65ebd
WB		 1292static unsigned int fanout_demux_lb(struct packet_fanout *f,
1293 struct sk_buff *skb,
1294 unsigned int num)
dc99f600
DM		 1295{
1296 int cur, old;
1297
1298 cur = atomic_read(&f->rr_cur);
1299 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1300 fanout_rr_next(f, num))) != cur)
1301 cur = old;
77f65ebd
WB		 1302 return cur;
1303}
1304
1305static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1306 struct sk_buff *skb,
1307 unsigned int num)
1308{
1309 return smp_processor_id() % num;
dc99f600
DM		 1310}
1311
5df0ddfb
DB		 1312static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1313 struct sk_buff *skb,
1314 unsigned int num)
1315{
f337db64 1316 return prandom_u32_max(num);
5df0ddfb
DB		 1317}
1318
77f65ebd
WB		 1319static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1320 struct sk_buff *skb,
ad377cab 1321 unsigned int idx, bool try_self,
77f65ebd 1322 unsigned int num)
95ec3eb4 1323{
0648ab70 1324 struct packet_sock *po;
77f65ebd 1325 unsigned int i, j;
95ec3eb4 1326
0648ab70
WB		 1327 po = pkt_sk(f->arr[idx]);
1328 if (try_self && packet_rcv_has_room(po, skb))
ad377cab
WB		 1329 return idx;
1330
0648ab70 1331 i = j = min_t(int, po->rollover->sock, num - 1);
77f65ebd 1332 do {
ad377cab 1333 if (i != idx && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
77f65ebd 1334 if (i != j)
0648ab70 1335 po->rollover->sock = i;
77f65ebd
WB		 1336 return i;
1337 }
ad377cab 1338
77f65ebd
WB		 1339 if (++i == num)
1340 i = 0;
1341 } while (i != j);
1342
1343 return idx;
1344}
1345
2d36097d
NH		 1346static unsigned int fanout_demux_qm(struct packet_fanout *f,
1347 struct sk_buff *skb,
1348 unsigned int num)
1349{
1350 return skb_get_queue_mapping(skb) % num;
1351}
1352
77f65ebd
WB		 1353static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1354{
1355 return f->flags & (flag >> 8);
95ec3eb4
DM		 1356}
1357
95ec3eb4
DM		 1358static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1359 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1360{
1361 struct packet_fanout *f = pt->af_packet_priv;
1362 unsigned int num = f->num_members;
1363 struct packet_sock *po;
77f65ebd 1364 unsigned int idx;
dc99f600
DM		 1365
1366 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1367 !num) {
1368 kfree_skb(skb);
1369 return 0;
1370 }
1371
3f34b24a
AD		 1372 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1373 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1374 if (!skb)
1375 return 0;
1376 }
95ec3eb4
DM		 1377 switch (f->type) {
1378 case PACKET_FANOUT_HASH:
1379 default:
77f65ebd 1380 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1381 break;
1382 case PACKET_FANOUT_LB:
77f65ebd 1383 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1384 break;
1385 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1386 idx = fanout_demux_cpu(f, skb, num);
1387 break;
5df0ddfb
DB		 1388 case PACKET_FANOUT_RND:
1389 idx = fanout_demux_rnd(f, skb, num);
1390 break;
2d36097d
NH		 1391 case PACKET_FANOUT_QM:
1392 idx = fanout_demux_qm(f, skb, num);
1393 break;
77f65ebd 1394 case PACKET_FANOUT_ROLLOVER:
ad377cab 1395 idx = fanout_demux_rollover(f, skb, 0, false, num);
95ec3eb4 1396 break;
dc99f600
DM		 1397 }
1398
ad377cab
WB		 1399 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1400 idx = fanout_demux_rollover(f, skb, idx, true, num);
dc99f600 1401
ad377cab 1402 po = pkt_sk(f->arr[idx]);
dc99f600
DM		 1403 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1404}
1405
fff3321d
PE		 1406DEFINE_MUTEX(fanout_mutex);
1407EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1408static LIST_HEAD(fanout_list);
1409
1410static void __fanout_link(struct sock *sk, struct packet_sock *po)
1411{
1412 struct packet_fanout *f = po->fanout;
1413
1414 spin_lock(&f->lock);
1415 f->arr[f->num_members] = sk;
1416 smp_wmb();
1417 f->num_members++;
1418 spin_unlock(&f->lock);
1419}
1420
1421static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1422{
1423 struct packet_fanout *f = po->fanout;
1424 int i;
1425
1426 spin_lock(&f->lock);
1427 for (i = 0; i < f->num_members; i++) {
1428 if (f->arr[i] == sk)
1429 break;
1430 }
1431 BUG_ON(i >= f->num_members);
1432 f->arr[i] = f->arr[f->num_members - 1];
1433 f->num_members--;
1434 spin_unlock(&f->lock);
1435}
1436
d4dd8aee 1437static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1438{
d4dd8aee 1439 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1440 return true;
1441
1442 return false;
1443}
1444
7736d33f 1445static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1446{
1447 struct packet_sock *po = pkt_sk(sk);
1448 struct packet_fanout *f, *match;
7736d33f 1449 u8 type = type_flags & 0xff;
77f65ebd 1450 u8 flags = type_flags >> 8;
dc99f600
DM		 1451 int err;
1452
1453 switch (type) {
77f65ebd
WB		 1454 case PACKET_FANOUT_ROLLOVER:
1455 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1456 return -EINVAL;
dc99f600
DM		 1457 case PACKET_FANOUT_HASH:
1458 case PACKET_FANOUT_LB:
95ec3eb4 1459 case PACKET_FANOUT_CPU:
5df0ddfb 1460 case PACKET_FANOUT_RND:
2d36097d 1461 case PACKET_FANOUT_QM:
dc99f600
DM		 1462 break;
1463 default:
1464 return -EINVAL;
1465 }
1466
1467 if (!po->running)
1468 return -EINVAL;
1469
1470 if (po->fanout)
1471 return -EALREADY;
1472
0648ab70
WB		 1473 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER) {
1474 po->rollover = kzalloc(sizeof(*po->rollover), GFP_KERNEL);
1475 if (!po->rollover)
1476 return -ENOMEM;
1477 }
1478
dc99f600
DM		 1479 mutex_lock(&fanout_mutex);
1480 match = NULL;
1481 list_for_each_entry(f, &fanout_list, list) {
1482 if (f->id == id &&
1483 read_pnet(&f->net) == sock_net(sk)) {
1484 match = f;
1485 break;
1486 }
1487 }
afe62c68 1488 err = -EINVAL;
77f65ebd 1489 if (match && match->flags != flags)
afe62c68 1490 goto out;
dc99f600 1491 if (!match) {
afe62c68 1492 err = -ENOMEM;
dc99f600 1493 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1494 if (!match)
1495 goto out;
1496 write_pnet(&match->net, sock_net(sk));
1497 match->id = id;
1498 match->type = type;
77f65ebd 1499 match->flags = flags;
afe62c68
ED		 1500 atomic_set(&match->rr_cur, 0);
1501 INIT_LIST_HEAD(&match->list);
1502 spin_lock_init(&match->lock);
1503 atomic_set(&match->sk_ref, 0);
1504 match->prot_hook.type = po->prot_hook.type;
1505 match->prot_hook.dev = po->prot_hook.dev;
1506 match->prot_hook.func = packet_rcv_fanout;
1507 match->prot_hook.af_packet_priv = match;
c0de08d0 1508 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1509 dev_add_pack(&match->prot_hook);
1510 list_add(&match->list, &fanout_list);
dc99f600 1511 }
afe62c68
ED		 1512 err = -EINVAL;
1513 if (match->type == type &&
1514 match->prot_hook.type == po->prot_hook.type &&
1515 match->prot_hook.dev == po->prot_hook.dev) {
1516 err = -ENOSPC;
1517 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1518 __dev_remove_pack(&po->prot_hook);
1519 po->fanout = match;
1520 atomic_inc(&match->sk_ref);
1521 __fanout_link(sk, po);
1522 err = 0;
dc99f600
DM		 1523 }
1524 }
afe62c68 1525out:
dc99f600 1526 mutex_unlock(&fanout_mutex);
0648ab70
WB		 1527 if (err) {
1528 kfree(po->rollover);
1529 po->rollover = NULL;
1530 }
dc99f600
DM		 1531 return err;
1532}
1533
1534static void fanout_release(struct sock *sk)
1535{
1536 struct packet_sock *po = pkt_sk(sk);
1537 struct packet_fanout *f;
1538
1539 f = po->fanout;
1540 if (!f)
1541 return;
1542
fff3321d 1543 mutex_lock(&fanout_mutex);
dc99f600
DM		 1544 po->fanout = NULL;
1545
dc99f600
DM		 1546 if (atomic_dec_and_test(&f->sk_ref)) {
1547 list_del(&f->list);
1548 dev_remove_pack(&f->prot_hook);
1549 kfree(f);
1550 }
1551 mutex_unlock(&fanout_mutex);
0648ab70
WB		 1552
1553 kfree(po->rollover);
dc99f600 1554}
1da177e4 1555
90ddc4f0 1556static const struct proto_ops packet_ops;
1da177e4 1557
90ddc4f0 1558static const struct proto_ops packet_ops_spkt;
1da177e4 1559
40d4e3df
ED		 1560static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1561 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1562{
1563 struct sock *sk;
1564 struct sockaddr_pkt *spkt;
1565
1566 /*
1567 * When we registered the protocol we saved the socket in the data
1568 * field for just this event.
1569 */
1570
1571 sk = pt->af_packet_priv;
1ce4f28b 1572
1da177e4
LT		 1573 /*
1574 * Yank back the headers [hope the device set this
1575 * right or kerboom...]
1576 *
1577 * Incoming packets have ll header pulled,
1578 * push it back.
1579 *
98e399f8 1580 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1581 * so that this procedure is noop.
1582 */
1583
1584 if (skb->pkt_type == PACKET_LOOPBACK)
1585 goto out;
1586
09ad9bc7 1587 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1588 goto out;
1589
40d4e3df
ED		 1590 skb = skb_share_check(skb, GFP_ATOMIC);
1591 if (skb == NULL)
1da177e4
LT		 1592 goto oom;
1593
1594 /* drop any routing info */
adf30907 1595 skb_dst_drop(skb);
1da177e4 1596
84531c24
PO		 1597 /* drop conntrack reference */
1598 nf_reset(skb);
1599
ffbc6111 1600 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1601
98e399f8 1602 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1603
1604 /*
1605 * The SOCK_PACKET socket receives _all_ frames.
1606 */
1607
1608 spkt->spkt_family = dev->type;
1609 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1610 spkt->spkt_protocol = skb->protocol;
1611
1612 /*
1613 * Charge the memory to the socket. This is done specifically
1614 * to prevent sockets using all the memory up.
1615 */
1616
40d4e3df 1617 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1618 return 0;
1619
1620out:
1621 kfree_skb(skb);
1622oom:
1623 return 0;
1624}
1625
1626
1627/*
1628 * Output a raw packet to a device layer. This bypasses all the other
1629 * protocol layers and you must therefore supply it with a complete frame
1630 */
1ce4f28b 1631
1b784140
YX		 1632static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1633 size_t len)
1da177e4
LT		 1634{
1635 struct sock *sk = sock->sk;
342dfc30 1636 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1637 struct sk_buff *skb = NULL;
1da177e4 1638 struct net_device *dev;
40d4e3df 1639 __be16 proto = 0;
1da177e4 1640 int err;
3bdc0eba 1641 int extra_len = 0;
1ce4f28b 1642
1da177e4 1643 /*
1ce4f28b 1644 * Get and verify the address.
1da177e4
LT		 1645 */
1646
40d4e3df 1647 if (saddr) {
1da177e4 1648 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1649 return -EINVAL;
1650 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1651 proto = saddr->spkt_protocol;
1652 } else
1653 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1654
1655 /*
1ce4f28b 1656 * Find the device first to size check it
1da177e4
LT		 1657 */
1658
de74e92a 1659 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1660retry:
654d1f8a
ED		 1661 rcu_read_lock();
1662 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1663 err = -ENODEV;
1664 if (dev == NULL)
1665 goto out_unlock;
1ce4f28b 1666
d5e76b0a
DM		 1667 err = -ENETDOWN;
1668 if (!(dev->flags & IFF_UP))
1669 goto out_unlock;
1670
1da177e4 1671 /*
40d4e3df
ED		 1672 * You may not queue a frame bigger than the mtu. This is the lowest level
1673 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1674 */
1ce4f28b 1675
3bdc0eba
BG		 1676 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1677 if (!netif_supports_nofcs(dev)) {
1678 err = -EPROTONOSUPPORT;
1679 goto out_unlock;
1680 }
1681 extra_len = 4; /* We're doing our own CRC */
1682 }
1683
1da177e4 1684 err = -EMSGSIZE;
3bdc0eba 1685 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1686 goto out_unlock;
1687
1a35ca80
ED		 1688 if (!skb) {
1689 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1690 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1691 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1692
1693 rcu_read_unlock();
4ce40912 1694 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1695 if (skb == NULL)
1696 return -ENOBUFS;
1697 /* FIXME: Save some space for broken drivers that write a hard
1698 * header at transmission time by themselves. PPP is the notable
1699 * one here. This should really be fixed at the driver level.
1700 */
1701 skb_reserve(skb, reserved);
1702 skb_reset_network_header(skb);
1703
1704 /* Try to align data part correctly */
1705 if (hhlen) {
1706 skb->data -= hhlen;
1707 skb->tail -= hhlen;
1708 if (len < hhlen)
1709 skb_reset_network_header(skb);
1710 }
6ce8e9ce 1711 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1a35ca80
ED		 1712 if (err)
1713 goto out_free;
1714 goto retry;
1da177e4
LT		 1715 }
1716
3bdc0eba 1717 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1718 /* Earlier code assumed this would be a VLAN pkt,
1719 * double-check this now that we have the actual
1720 * packet in hand.
1721 */
1722 struct ethhdr *ehdr;
1723 skb_reset_mac_header(skb);
1724 ehdr = eth_hdr(skb);
1725 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1726 err = -EMSGSIZE;
1727 goto out_unlock;
1728 }
1729 }
1a35ca80 1730
1da177e4
LT		 1731 skb->protocol = proto;
1732 skb->dev = dev;
1733 skb->priority = sk->sk_priority;
2d37a186 1734 skb->mark = sk->sk_mark;
bf84a010
DB		 1735
1736 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1737
3bdc0eba
BG		 1738 if (unlikely(extra_len == 4))
1739 skb->no_fcs = 1;
1740
40893fd0 1741 skb_probe_transport_header(skb, 0);
c1aad275 1742
1da177e4 1743 dev_queue_xmit(skb);
654d1f8a 1744 rcu_read_unlock();
40d4e3df 1745 return len;
1da177e4 1746
1da177e4 1747out_unlock:
654d1f8a 1748 rcu_read_unlock();
1a35ca80
ED		 1749out_free:
1750 kfree_skb(skb);
1da177e4
LT		 1751 return err;
1752}
1da177e4 1753
eea49cc9 1754static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1755 const struct sock *sk,
dbcb5855 1756 unsigned int res)
1da177e4
LT		 1757{
1758 struct sk_filter *filter;
fda9ef5d 1759
80f8f102
ED		 1760 rcu_read_lock();
1761 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1762 if (filter != NULL)
0a14842f 1763 res = SK_RUN_FILTER(filter, skb);
80f8f102 1764 rcu_read_unlock();
1da177e4 1765
dbcb5855 1766 return res;
1da177e4
LT		 1767}
1768
1769/*
62ab0812
ED		 1770 * This function makes lazy skb cloning in hope that most of packets
1771 * are discarded by BPF.
1772 *
1773 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1774 * and skb->cb are mangled. It works because (and until) packets
1775 * falling here are owned by current CPU. Output packets are cloned
1776 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1777 * sequencially, so that if we return skb to original state on exit,
1778 * we will not harm anyone.
1da177e4
LT		 1779 */
1780
40d4e3df
ED		 1781static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1782 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1783{
1784 struct sock *sk;
1785 struct sockaddr_ll *sll;
1786 struct packet_sock *po;
40d4e3df 1787 u8 *skb_head = skb->data;
1da177e4 1788 int skb_len = skb->len;
dbcb5855 1789 unsigned int snaplen, res;
1da177e4
LT		 1790
1791 if (skb->pkt_type == PACKET_LOOPBACK)
1792 goto drop;
1793
1794 sk = pt->af_packet_priv;
1795 po = pkt_sk(sk);
1796
09ad9bc7 1797 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1798 goto drop;
1799
1da177e4
LT		 1800 skb->dev = dev;
1801
3b04ddde 1802 if (dev->header_ops) {
1da177e4 1803 /* The device has an explicit notion of ll header,
62ab0812
ED		 1804 * exported to higher levels.
1805 *
1806 * Otherwise, the device hides details of its frame
1807 * structure, so that corresponding packet head is
1808 * never delivered to user.
1da177e4
LT		 1809 */
1810 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1811 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1812 else if (skb->pkt_type == PACKET_OUTGOING) {
1813 /* Special case: outgoing packets have ll header at head */
bbe735e4 1814 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1815 }
1816 }
1817
1818 snaplen = skb->len;
1819
dbcb5855
DM		 1820 res = run_filter(skb, sk, snaplen);
1821 if (!res)
fda9ef5d 1822 goto drop_n_restore;
dbcb5855
DM		 1823 if (snaplen > res)
1824 snaplen = res;
1da177e4 1825
0fd7bac6 1826 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1827 goto drop_n_acct;
1828
1829 if (skb_shared(skb)) {
1830 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1831 if (nskb == NULL)
1832 goto drop_n_acct;
1833
1834 if (skb_head != skb->data) {
1835 skb->data = skb_head;
1836 skb->len = skb_len;
1837 }
abc4e4fa 1838 consume_skb(skb);
1da177e4
LT		 1839 skb = nskb;
1840 }
1841
b4772ef8 1842 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
ffbc6111
HX		 1843
1844 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4 1845 sll->sll_hatype = dev->type;
1da177e4 1846 sll->sll_pkttype = skb->pkt_type;
8032b464 1847 if (unlikely(po->origdev))
80feaacb
PWJ		 1848 sll->sll_ifindex = orig_dev->ifindex;
1849 else
1850 sll->sll_ifindex = dev->ifindex;
1da177e4 1851
b95cce35 1852 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1853
2472d761
EB		 1854 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
1855 * Use their space for storing the original skb length.
1856 */
1857 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
8dc41944 1858
1da177e4
LT		 1859 if (pskb_trim(skb, snaplen))
1860 goto drop_n_acct;
1861
1862 skb_set_owner_r(skb, sk);
1863 skb->dev = NULL;
adf30907 1864 skb_dst_drop(skb);
1da177e4 1865
84531c24
PO		 1866 /* drop conntrack reference */
1867 nf_reset(skb);
1868
1da177e4 1869 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1870 po->stats.stats1.tp_packets++;
3bc3b96f 1871 sock_skb_set_dropcount(sk, skb);
1da177e4
LT		 1872 __skb_queue_tail(&sk->sk_receive_queue, skb);
1873 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 1874 sk->sk_data_ready(sk);
1da177e4
LT		 1875 return 0;
1876
1877drop_n_acct:
7091fbd8 1878 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1879 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1880 atomic_inc(&sk->sk_drops);
1881 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1882
1883drop_n_restore:
1884 if (skb_head != skb->data && skb_shared(skb)) {
1885 skb->data = skb_head;
1886 skb->len = skb_len;
1887 }
1888drop:
ead2ceb0 1889 consume_skb(skb);
1da177e4
LT		 1890 return 0;
1891}
1892
40d4e3df
ED		 1893static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1894 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1895{
1896 struct sock *sk;
1897 struct packet_sock *po;
1898 struct sockaddr_ll *sll;
184f489e 1899 union tpacket_uhdr h;
40d4e3df 1900 u8 *skb_head = skb->data;
1da177e4 1901 int skb_len = skb->len;
dbcb5855 1902 unsigned int snaplen, res;
f6fb8f10 1903 unsigned long status = TP_STATUS_USER;
bbd6ef87 1904 unsigned short macoff, netoff, hdrlen;
1da177e4 1905 struct sk_buff *copy_skb = NULL;
bbd6ef87 1906 struct timespec ts;
b9c32fb2 1907 __u32 ts_status;
1da177e4 1908
51846355
AW		 1909 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1910 * We may add members to them until current aligned size without forcing
1911 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1912 */
1913 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1914 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1915
1da177e4
LT		 1916 if (skb->pkt_type == PACKET_LOOPBACK)
1917 goto drop;
1918
1919 sk = pt->af_packet_priv;
1920 po = pkt_sk(sk);
1921
09ad9bc7 1922 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1923 goto drop;
1924
3b04ddde 1925 if (dev->header_ops) {
1da177e4 1926 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1927 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1928 else if (skb->pkt_type == PACKET_OUTGOING) {
1929 /* Special case: outgoing packets have ll header at head */
bbe735e4 1930 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1931 }
1932 }
1933
1934 snaplen = skb->len;
1935
dbcb5855
DM		 1936 res = run_filter(skb, sk, snaplen);
1937 if (!res)
fda9ef5d 1938 goto drop_n_restore;
68c2e5de
AD		 1939
1940 if (skb->ip_summed == CHECKSUM_PARTIAL)
1941 status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 1942 else if (skb->pkt_type != PACKET_OUTGOING &&
1943 (skb->ip_summed == CHECKSUM_COMPLETE ||
1944 skb_csum_unnecessary(skb)))
1945 status |= TP_STATUS_CSUM_VALID;
68c2e5de 1946
dbcb5855
DM		 1947 if (snaplen > res)
1948 snaplen = res;
1da177e4
LT		 1949
1950 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1951 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1952 po->tp_reserve;
1da177e4 1953 } else {
95c96174 1954 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1955 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1956 (maclen < 16 ? 16 : maclen)) +
1957 po->tp_reserve;
1da177e4
LT		 1958 macoff = netoff - maclen;
1959 }
f6fb8f10 1960 if (po->tp_version <= TPACKET_V2) {
1961 if (macoff + snaplen > po->rx_ring.frame_size) {
1962 if (po->copy_thresh &&
0fd7bac6 1963 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1964 if (skb_shared(skb)) {
1965 copy_skb = skb_clone(skb, GFP_ATOMIC);
1966 } else {
1967 copy_skb = skb_get(skb);
1968 skb_head = skb->data;
1969 }
1970 if (copy_skb)
1971 skb_set_owner_r(copy_skb, sk);
1da177e4 1972 }
f6fb8f10 1973 snaplen = po->rx_ring.frame_size - macoff;
1974 if ((int)snaplen < 0)
1975 snaplen = 0;
1da177e4 1976 }
dc808110
ED		 1977 } else if (unlikely(macoff + snaplen >
1978 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1979 u32 nval;
1980
1981 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1982 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1983 snaplen, nval, macoff);
1984 snaplen = nval;
1985 if (unlikely((int)snaplen < 0)) {
1986 snaplen = 0;
1987 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1988 }
1da177e4 1989 }
1da177e4 1990 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1991 h.raw = packet_current_rx_frame(po, skb,
1992 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1993 if (!h.raw)
1da177e4 1994 goto ring_is_full;
f6fb8f10 1995 if (po->tp_version <= TPACKET_V2) {
1996 packet_increment_rx_head(po, &po->rx_ring);
1997 /*
1998 * LOSING will be reported till you read the stats,
1999 * because it's COR - Clear On Read.
2000 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2001 * at packet level.
2002 */
ee80fbf3 2003 if (po->stats.stats1.tp_drops)
f6fb8f10 2004 status |= TP_STATUS_LOSING;
2005 }
ee80fbf3 2006 po->stats.stats1.tp_packets++;
1da177e4
LT		 2007 if (copy_skb) {
2008 status |= TP_STATUS_COPY;
2009 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2010 }
1da177e4
LT		 2011 spin_unlock(&sk->sk_receive_queue.lock);
2012
bbd6ef87 2013 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 2014
2015 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 2016 getnstimeofday(&ts);
1da177e4 2017
b9c32fb2
DB		 2018 status |= ts_status;
2019
bbd6ef87
PM		 2020 switch (po->tp_version) {
2021 case TPACKET_V1:
2022 h.h1->tp_len = skb->len;
2023 h.h1->tp_snaplen = snaplen;
2024 h.h1->tp_mac = macoff;
2025 h.h1->tp_net = netoff;
4b457bdf
DB		 2026 h.h1->tp_sec = ts.tv_sec;
2027 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 2028 hdrlen = sizeof(*h.h1);
2029 break;
2030 case TPACKET_V2:
2031 h.h2->tp_len = skb->len;
2032 h.h2->tp_snaplen = snaplen;
2033 h.h2->tp_mac = macoff;
2034 h.h2->tp_net = netoff;
bbd6ef87
PM		 2035 h.h2->tp_sec = ts.tv_sec;
2036 h.h2->tp_nsec = ts.tv_nsec;
df8a39de
JP		 2037 if (skb_vlan_tag_present(skb)) {
2038 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 2039 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2040 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2041 } else {
2042 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2043 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2044 }
e4d26f4b 2045 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2046 hdrlen = sizeof(*h.h2);
2047 break;
f6fb8f10 2048 case TPACKET_V3:
2049 /* tp_nxt_offset,vlan are already populated above.
2050 * So DONT clear those fields here
2051 */
2052 h.h3->tp_status |= status;
2053 h.h3->tp_len = skb->len;
2054 h.h3->tp_snaplen = snaplen;
2055 h.h3->tp_mac = macoff;
2056 h.h3->tp_net = netoff;
f6fb8f10 2057 h.h3->tp_sec = ts.tv_sec;
2058 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2059 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2060 hdrlen = sizeof(*h.h3);
2061 break;
bbd6ef87
PM		 2062 default:
2063 BUG();
2064 }
1da177e4 2065
bbd6ef87 2066 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2067 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2068 sll->sll_family = AF_PACKET;
2069 sll->sll_hatype = dev->type;
2070 sll->sll_protocol = skb->protocol;
2071 sll->sll_pkttype = skb->pkt_type;
8032b464 2072 if (unlikely(po->origdev))
80feaacb
PWJ		 2073 sll->sll_ifindex = orig_dev->ifindex;
2074 else
2075 sll->sll_ifindex = dev->ifindex;
1da177e4 2076
e16aa207 2077 smp_mb();
f0d4eb29 2078
f6dafa95 2079#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2080 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2081 u8 *start, *end;
2082
f0d4eb29
DB		 2083 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2084 macoff + snaplen);
2085
2086 for (start = h.raw; start < end; start += PAGE_SIZE)
2087 flush_dcache_page(pgv_to_page(start));
1da177e4 2088 }
f0d4eb29 2089 smp_wmb();
f6dafa95 2090#endif
f0d4eb29 2091
da413eec 2092 if (po->tp_version <= TPACKET_V2) {
f6fb8f10 2093 __packet_set_status(po, h.raw, status);
da413eec
DC		 2094 sk->sk_data_ready(sk);
2095 } else {
f6fb8f10 2096 prb_clear_blk_fill_status(&po->rx_ring);
da413eec 2097 }
1da177e4
LT		 2098
2099drop_n_restore:
2100 if (skb_head != skb->data && skb_shared(skb)) {
2101 skb->data = skb_head;
2102 skb->len = skb_len;
2103 }
2104drop:
1ce4f28b 2105 kfree_skb(skb);
1da177e4
LT		 2106 return 0;
2107
2108ring_is_full:
ee80fbf3 2109 po->stats.stats1.tp_drops++;
1da177e4
LT		 2110 spin_unlock(&sk->sk_receive_queue.lock);
2111
676d2369 2112 sk->sk_data_ready(sk);
acb5d75b 2113 kfree_skb(copy_skb);
1da177e4
LT		 2114 goto drop_n_restore;
2115}
2116
69e3c75f
JB		 2117static void tpacket_destruct_skb(struct sk_buff *skb)
2118{
2119 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2120
69e3c75f 2121 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2122 void *ph;
b9c32fb2
DB		 2123 __u32 ts;
2124
69e3c75f 2125 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2126 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2127
2128 ts = __packet_set_timestamp(po, ph, skb);
2129 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2130 }
2131
2132 sock_wfree(skb);
2133}
2134
9c707762
WB		 2135static bool ll_header_truncated(const struct net_device *dev, int len)
2136{
2137 /* net device doesn't like empty head */
2138 if (unlikely(len <= dev->hard_header_len)) {
eee2f04b 2139 net_warn_ratelimited("%s: packet size is too short (%d <= %d)\n",
9c707762
WB		 2140 current->comm, len, dev->hard_header_len);
2141 return true;
2142 }
2143
2144 return false;
2145}
2146
40d4e3df
ED		 2147static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2148 void *frame, struct net_device *dev, int size_max,
ae641949 2149 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2150{
184f489e 2151 union tpacket_uhdr ph;
09effa67 2152 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2153 struct socket *sock = po->sk.sk_socket;
2154 struct page *page;
2155 void *data;
2156 int err;
2157
2158 ph.raw = frame;
2159
2160 skb->protocol = proto;
2161 skb->dev = dev;
2162 skb->priority = po->sk.sk_priority;
2d37a186 2163 skb->mark = po->sk.sk_mark;
2e31396f 2164 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2165 skb_shinfo(skb)->destructor_arg = ph.raw;
2166
2167 switch (po->tp_version) {
2168 case TPACKET_V2:
2169 tp_len = ph.h2->tp_len;
2170 break;
2171 default:
2172 tp_len = ph.h1->tp_len;
2173 break;
2174 }
09effa67
DM		 2175 if (unlikely(tp_len > size_max)) {
2176 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2177 return -EMSGSIZE;
2178 }
69e3c75f 2179
ae641949 2180 skb_reserve(skb, hlen);
69e3c75f 2181 skb_reset_network_header(skb);
c1aad275 2182
d346a3fa
DB		 2183 if (!packet_use_direct_xmit(po))
2184 skb_probe_transport_header(skb, 0);
2185 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2186 int off_min, off_max, off;
2187 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2188 off_max = po->tx_ring.frame_size - tp_len;
2189 if (sock->type == SOCK_DGRAM) {
2190 switch (po->tp_version) {
2191 case TPACKET_V2:
2192 off = ph.h2->tp_net;
2193 break;
2194 default:
2195 off = ph.h1->tp_net;
2196 break;
2197 }
2198 } else {
2199 switch (po->tp_version) {
2200 case TPACKET_V2:
2201 off = ph.h2->tp_mac;
2202 break;
2203 default:
2204 off = ph.h1->tp_mac;
2205 break;
2206 }
2207 }
2208 if (unlikely((off < off_min) || (off_max < off)))
2209 return -EINVAL;
2210 data = ph.raw + off;
2211 } else {
2212 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2213 }
69e3c75f
JB		 2214 to_write = tp_len;
2215
2216 if (sock->type == SOCK_DGRAM) {
2217 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2218 NULL, tp_len);
2219 if (unlikely(err < 0))
2220 return -EINVAL;
40d4e3df 2221 } else if (dev->hard_header_len) {
9c707762 2222 if (ll_header_truncated(dev, tp_len))
69e3c75f 2223 return -EINVAL;
69e3c75f
JB		 2224
2225 skb_push(skb, dev->hard_header_len);
2226 err = skb_store_bits(skb, 0, data,
2227 dev->hard_header_len);
2228 if (unlikely(err))
2229 return err;
2230
2231 data += dev->hard_header_len;
2232 to_write -= dev->hard_header_len;
2233 }
2234
69e3c75f
JB		 2235 offset = offset_in_page(data);
2236 len_max = PAGE_SIZE - offset;
2237 len = ((to_write > len_max) ? len_max : to_write);
2238
2239 skb->data_len = to_write;
2240 skb->len += to_write;
2241 skb->truesize += to_write;
2242 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2243
2244 while (likely(to_write)) {
2245 nr_frags = skb_shinfo(skb)->nr_frags;
2246
2247 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2248 pr_err("Packet exceed the number of skb frags(%lu)\n",
2249 MAX_SKB_FRAGS);
69e3c75f
JB		 2250 return -EFAULT;
2251 }
2252
0af55bb5
CG		 2253 page = pgv_to_page(data);
2254 data += len;
69e3c75f
JB		 2255 flush_dcache_page(page);
2256 get_page(page);
0af55bb5 2257 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2258 to_write -= len;
2259 offset = 0;
2260 len_max = PAGE_SIZE;
2261 len = ((to_write > len_max) ? len_max : to_write);
2262 }
2263
2264 return tp_len;
2265}
2266
2267static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2268{
69e3c75f
JB		 2269 struct sk_buff *skb;
2270 struct net_device *dev;
2271 __be16 proto;
09effa67 2272 int err, reserve = 0;
40d4e3df 2273 void *ph;
342dfc30 2274 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2275 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2276 int tp_len, size_max;
2277 unsigned char *addr;
2278 int len_sum = 0;
9e67030a 2279 int status = TP_STATUS_AVAILABLE;
ae641949 2280 int hlen, tlen;
69e3c75f 2281
69e3c75f
JB		 2282 mutex_lock(&po->pg_vec_lock);
2283
66e56cd4 2284 if (likely(saddr == NULL)) {
e40526cb 2285 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2286 proto = po->num;
2287 addr = NULL;
2288 } else {
2289 err = -EINVAL;
2290 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2291 goto out;
2292 if (msg->msg_namelen < (saddr->sll_halen
2293 + offsetof(struct sockaddr_ll,
2294 sll_addr)))
2295 goto out;
69e3c75f
JB		 2296 proto = saddr->sll_protocol;
2297 addr = saddr->sll_addr;
827d9780 2298 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2299 }
2300
69e3c75f
JB		 2301 err = -ENXIO;
2302 if (unlikely(dev == NULL))
2303 goto out;
69e3c75f
JB		 2304 err = -ENETDOWN;
2305 if (unlikely(!(dev->flags & IFF_UP)))
2306 goto out_put;
2307
52f1454f 2308 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2309 size_max = po->tx_ring.frame_size
b5dd884e 2310 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2311
09effa67
DM		 2312 if (size_max > dev->mtu + reserve)
2313 size_max = dev->mtu + reserve;
2314
69e3c75f
JB		 2315 do {
2316 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2317 TP_STATUS_SEND_REQUEST);
69e3c75f 2318 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2319 if (need_wait && need_resched())
2320 schedule();
69e3c75f
JB		 2321 continue;
2322 }
2323
2324 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2325 hlen = LL_RESERVED_SPACE(dev);
2326 tlen = dev->needed_tailroom;
69e3c75f 2327 skb = sock_alloc_send_skb(&po->sk,
ae641949 2328 hlen + tlen + sizeof(struct sockaddr_ll),
fbf33a28 2329 !need_wait, &err);
69e3c75f 2330
fbf33a28
KM		 2331 if (unlikely(skb == NULL)) {
2332 /* we assume the socket was initially writeable ... */
2333 if (likely(len_sum > 0))
2334 err = len_sum;
69e3c75f 2335 goto out_status;
fbf33a28 2336 }
69e3c75f 2337 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f
DB		 2338 addr, hlen);
2339 if (tp_len > dev->mtu + dev->hard_header_len) {
2340 struct ethhdr *ehdr;
2341 /* Earlier code assumed this would be a VLAN pkt,
2342 * double-check this now that we have the actual
2343 * packet in hand.
2344 */
69e3c75f 2345
52f1454f
DB		 2346 skb_reset_mac_header(skb);
2347 ehdr = eth_hdr(skb);
2348 if (ehdr->h_proto != htons(ETH_P_8021Q))
2349 tp_len = -EMSGSIZE;
2350 }
69e3c75f
JB		 2351 if (unlikely(tp_len < 0)) {
2352 if (po->tp_loss) {
2353 __packet_set_status(po, ph,
2354 TP_STATUS_AVAILABLE);
2355 packet_increment_head(&po->tx_ring);
2356 kfree_skb(skb);
2357 continue;
2358 } else {
2359 status = TP_STATUS_WRONG_FORMAT;
2360 err = tp_len;
2361 goto out_status;
2362 }
2363 }
2364
0fd5d57b
DB		 2365 packet_pick_tx_queue(dev, skb);
2366
69e3c75f
JB		 2367 skb->destructor = tpacket_destruct_skb;
2368 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2369 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2370
2371 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2372 err = po->xmit(skb);
eb70df13
JP		 2373 if (unlikely(err > 0)) {
2374 err = net_xmit_errno(err);
2375 if (err && __packet_get_status(po, ph) ==
2376 TP_STATUS_AVAILABLE) {
2377 /* skb was destructed already */
2378 skb = NULL;
2379 goto out_status;
2380 }
2381 /*
2382 * skb was dropped but not destructed yet;
2383 * let's treat it like congestion or err < 0
2384 */
2385 err = 0;
2386 }
69e3c75f
JB		 2387 packet_increment_head(&po->tx_ring);
2388 len_sum += tp_len;
b0138408
DB		 2389 } while (likely((ph != NULL) ||
2390 /* Note: packet_read_pending() might be slow if we have
2391 * to call it as it's per_cpu variable, but in fast-path
2392 * we already short-circuit the loop with the first
2393 * condition, and luckily don't have to go that path
2394 * anyway.
2395 */
2396 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2397
2398 err = len_sum;
2399 goto out_put;
2400
69e3c75f
JB		 2401out_status:
2402 __packet_set_status(po, ph, status);
2403 kfree_skb(skb);
2404out_put:
e40526cb 2405 dev_put(dev);
69e3c75f
JB		 2406out:
2407 mutex_unlock(&po->pg_vec_lock);
2408 return err;
2409}
69e3c75f 2410
eea49cc9
OJ		 2411static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2412 size_t reserve, size_t len,
2413 size_t linear, int noblock,
2414 int *err)
bfd5f4a3
SS		 2415{
2416 struct sk_buff *skb;
2417
2418 /* Under a page? Don't bother with paged skb. */
2419 if (prepad + len < PAGE_SIZE || !linear)
2420 linear = len;
2421
2422 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2423 err, 0);
bfd5f4a3
SS		 2424 if (!skb)
2425 return NULL;
2426
2427 skb_reserve(skb, reserve);
2428 skb_put(skb, linear);
2429 skb->data_len = len - linear;
2430 skb->len += len - linear;
2431
2432 return skb;
2433}
2434
d346a3fa 2435static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2436{
2437 struct sock *sk = sock->sk;
342dfc30 2438 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2439 struct sk_buff *skb;
2440 struct net_device *dev;
0e11c91e 2441 __be16 proto;
1da177e4 2442 unsigned char *addr;
827d9780 2443 int err, reserve = 0;
bfd5f4a3
SS		 2444 struct virtio_net_hdr vnet_hdr = { 0 };
2445 int offset = 0;
2446 int vnet_hdr_len;
2447 struct packet_sock *po = pkt_sk(sk);
2448 unsigned short gso_type = 0;
ae641949 2449 int hlen, tlen;
3bdc0eba 2450 int extra_len = 0;
8feb2fb2 2451 ssize_t n;
1da177e4
LT		 2452
2453 /*
1ce4f28b 2454 * Get and verify the address.
1da177e4 2455 */
1ce4f28b 2456
66e56cd4 2457 if (likely(saddr == NULL)) {
e40526cb 2458 dev = packet_cached_dev_get(po);
1da177e4
LT		 2459 proto = po->num;
2460 addr = NULL;
2461 } else {
2462 err = -EINVAL;
2463 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2464 goto out;
0fb375fb
EB		 2465 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2466 goto out;
1da177e4
LT		 2467 proto = saddr->sll_protocol;
2468 addr = saddr->sll_addr;
827d9780 2469 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2470 }
2471
1da177e4 2472 err = -ENXIO;
e40526cb 2473 if (unlikely(dev == NULL))
1da177e4 2474 goto out_unlock;
d5e76b0a 2475 err = -ENETDOWN;
e40526cb 2476 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2477 goto out_unlock;
2478
e40526cb
DB		 2479 if (sock->type == SOCK_RAW)
2480 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2481 if (po->has_vnet_hdr) {
2482 vnet_hdr_len = sizeof(vnet_hdr);
2483
2484 err = -EINVAL;
2485 if (len < vnet_hdr_len)
2486 goto out_unlock;
2487
2488 len -= vnet_hdr_len;
2489
8feb2fb2 2490 err = -EFAULT;
c0371da6 2491 n = copy_from_iter(&vnet_hdr, vnet_hdr_len, &msg->msg_iter);
8feb2fb2 2492 if (n != vnet_hdr_len)
bfd5f4a3
SS		 2493 goto out_unlock;
2494
2495 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
dc9e5153
MT		 2496 (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
2497 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
2498 __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
2499 vnet_hdr.hdr_len = __cpu_to_virtio16(false,
2500 __virtio16_to_cpu(false, vnet_hdr.csum_start) +
2501 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
bfd5f4a3
SS		 2502
2503 err = -EINVAL;
dc9e5153 2504 if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
bfd5f4a3
SS		 2505 goto out_unlock;
2506
2507 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2508 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2509 case VIRTIO_NET_HDR_GSO_TCPV4:
2510 gso_type = SKB_GSO_TCPV4;
2511 break;
2512 case VIRTIO_NET_HDR_GSO_TCPV6:
2513 gso_type = SKB_GSO_TCPV6;
2514 break;
2515 case VIRTIO_NET_HDR_GSO_UDP:
2516 gso_type = SKB_GSO_UDP;
2517 break;
2518 default:
2519 goto out_unlock;
2520 }
2521
2522 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2523 gso_type |= SKB_GSO_TCP_ECN;
2524
2525 if (vnet_hdr.gso_size == 0)
2526 goto out_unlock;
2527
2528 }
2529 }
2530
3bdc0eba
BG		 2531 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2532 if (!netif_supports_nofcs(dev)) {
2533 err = -EPROTONOSUPPORT;
2534 goto out_unlock;
2535 }
2536 extra_len = 4; /* We're doing our own CRC */
2537 }
2538
1da177e4 2539 err = -EMSGSIZE;
3bdc0eba 2540 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2541 goto out_unlock;
2542
bfd5f4a3 2543 err = -ENOBUFS;
ae641949
HX		 2544 hlen = LL_RESERVED_SPACE(dev);
2545 tlen = dev->needed_tailroom;
dc9e5153
MT		 2546 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2547 __virtio16_to_cpu(false, vnet_hdr.hdr_len),
bfd5f4a3 2548 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2549 if (skb == NULL)
1da177e4
LT		 2550 goto out_unlock;
2551
bfd5f4a3 2552 skb_set_network_header(skb, reserve);
1da177e4 2553
0c4e8581 2554 err = -EINVAL;
9c707762
WB		 2555 if (sock->type == SOCK_DGRAM) {
2556 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
46d2cfb1 2557 if (unlikely(offset < 0))
9c707762
WB		 2558 goto out_free;
2559 } else {
2560 if (ll_header_truncated(dev, len))
2561 goto out_free;
2562 }
1da177e4
LT		 2563
2564 /* Returns -EFAULT on error */
c0371da6 2565 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
1da177e4
LT		 2566 if (err)
2567 goto out_free;
bf84a010
DB		 2568
2569 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2570
3bdc0eba 2571 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2572 /* Earlier code assumed this would be a VLAN pkt,
2573 * double-check this now that we have the actual
2574 * packet in hand.
2575 */
2576 struct ethhdr *ehdr;
2577 skb_reset_mac_header(skb);
2578 ehdr = eth_hdr(skb);
2579 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2580 err = -EMSGSIZE;
2581 goto out_free;
2582 }
57f89bfa
BG		 2583 }
2584
09effa67
DM		 2585 skb->protocol = proto;
2586 skb->dev = dev;
1da177e4 2587 skb->priority = sk->sk_priority;
2d37a186 2588 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2589
2590 packet_pick_tx_queue(dev, skb);
1da177e4 2591
bfd5f4a3
SS		 2592 if (po->has_vnet_hdr) {
2593 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
dc9e5153
MT		 2594 u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
2595 u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
2596 if (!skb_partial_csum_set(skb, s, o)) {
bfd5f4a3
SS		 2597 err = -EINVAL;
2598 goto out_free;
2599 }
2600 }
2601
dc9e5153
MT		 2602 skb_shinfo(skb)->gso_size =
2603 __virtio16_to_cpu(false, vnet_hdr.gso_size);
bfd5f4a3
SS		 2604 skb_shinfo(skb)->gso_type = gso_type;
2605
2606 /* Header must be checked, and gso_segs computed. */
2607 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2608 skb_shinfo(skb)->gso_segs = 0;
2609
2610 len += vnet_hdr_len;
2611 }
2612
d346a3fa
DB		 2613 if (!packet_use_direct_xmit(po))
2614 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2615 if (unlikely(extra_len == 4))
2616 skb->no_fcs = 1;
2617
d346a3fa 2618 err = po->xmit(skb);
1da177e4
LT		 2619 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2620 goto out_unlock;
2621
e40526cb 2622 dev_put(dev);
1da177e4 2623
40d4e3df 2624 return len;
1da177e4
LT		 2625
2626out_free:
2627 kfree_skb(skb);
2628out_unlock:
e40526cb 2629 if (dev)
1da177e4
LT		 2630 dev_put(dev);
2631out:
2632 return err;
2633}
2634
1b784140 2635static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
69e3c75f 2636{
69e3c75f
JB		 2637 struct sock *sk = sock->sk;
2638 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2639
69e3c75f
JB		 2640 if (po->tx_ring.pg_vec)
2641 return tpacket_snd(po, msg);
2642 else
69e3c75f
JB		 2643 return packet_snd(sock, msg, len);
2644}
2645
1da177e4
LT		 2646/*
2647 * Close a PACKET socket. This is fairly simple. We immediately go
2648 * to 'closed' state and remove our protocol entry in the device list.
2649 */
2650
2651static int packet_release(struct socket *sock)
2652{
2653 struct sock *sk = sock->sk;
2654 struct packet_sock *po;
d12d01d6 2655 struct net *net;
f6fb8f10 2656 union tpacket_req_u req_u;
1da177e4
LT		 2657
2658 if (!sk)
2659 return 0;
2660
3b1e0a65 2661 net = sock_net(sk);
1da177e4
LT		 2662 po = pkt_sk(sk);
2663
0fa7fa98 2664 mutex_lock(&net->packet.sklist_lock);
808f5114 2665 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2666 mutex_unlock(&net->packet.sklist_lock);
2667
2668 preempt_disable();
920de804 2669 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2670 preempt_enable();
1da177e4 2671
808f5114 2672 spin_lock(&po->bind_lock);
ce06b03e 2673 unregister_prot_hook(sk, false);
66e56cd4
DB		 2674 packet_cached_dev_reset(po);
2675
160ff18a
BG		 2676 if (po->prot_hook.dev) {
2677 dev_put(po->prot_hook.dev);
2678 po->prot_hook.dev = NULL;
2679 }
808f5114 2680 spin_unlock(&po->bind_lock);
1da177e4 2681
1da177e4 2682 packet_flush_mclist(sk);
1da177e4 2683
9665d5d6
PS		 2684 if (po->rx_ring.pg_vec) {
2685 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2686 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2687 }
69e3c75f 2688
9665d5d6
PS		 2689 if (po->tx_ring.pg_vec) {
2690 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2691 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2692 }
1da177e4 2693
dc99f600
DM		 2694 fanout_release(sk);
2695
808f5114 2696 synchronize_net();
1da177e4
LT		 2697 /*
2698 * Now the socket is dead. No more input will appear.
2699 */
1da177e4
LT		 2700 sock_orphan(sk);
2701 sock->sk = NULL;
2702
2703 /* Purge queues */
2704
2705 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2706 packet_free_pending(po);
17ab56a2 2707 sk_refcnt_debug_release(sk);
1da177e4
LT		 2708
2709 sock_put(sk);
2710 return 0;
2711}
2712
2713/*
2714 * Attach a packet hook.
2715 */
2716
902fefb8 2717static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2718{
2719 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2720 const struct net_device *dev_curr;
2721 __be16 proto_curr;
2722 bool need_rehook;
dc99f600 2723
aef950b4
WY		 2724 if (po->fanout) {
2725 if (dev)
2726 dev_put(dev);
2727
dc99f600 2728 return -EINVAL;
aef950b4 2729 }
1da177e4
LT		 2730
2731 lock_sock(sk);
1da177e4 2732 spin_lock(&po->bind_lock);
66e56cd4 2733
902fefb8
DB		 2734 proto_curr = po->prot_hook.type;
2735 dev_curr = po->prot_hook.dev;
2736
2737 need_rehook = proto_curr != proto || dev_curr != dev;
2738
2739 if (need_rehook) {
2740 unregister_prot_hook(sk, true);
1da177e4 2741
902fefb8
DB		 2742 po->num = proto;
2743 po->prot_hook.type = proto;
1da177e4 2744
902fefb8
DB		 2745 if (po->prot_hook.dev)
2746 dev_put(po->prot_hook.dev);
2747
2748 po->prot_hook.dev = dev;
2749
2750 po->ifindex = dev ? dev->ifindex : 0;
2751 packet_cached_dev_assign(po, dev);
2752 }
66e56cd4 2753
902fefb8 2754 if (proto == 0 || !need_rehook)
1da177e4
LT		 2755 goto out_unlock;
2756
be85d4ad 2757 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2758 register_prot_hook(sk);
be85d4ad
UT		 2759 } else {
2760 sk->sk_err = ENETDOWN;
2761 if (!sock_flag(sk, SOCK_DEAD))
2762 sk->sk_error_report(sk);
1da177e4
LT		 2763 }
2764
2765out_unlock:
2766 spin_unlock(&po->bind_lock);
2767 release_sock(sk);
2768 return 0;
2769}
2770
2771/*
2772 * Bind a packet socket to a device
2773 */
2774
40d4e3df
ED		 2775static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2776 int addr_len)
1da177e4 2777{
40d4e3df 2778 struct sock *sk = sock->sk;
1da177e4
LT		 2779 char name[15];
2780 struct net_device *dev;
2781 int err = -ENODEV;
1ce4f28b 2782
1da177e4
LT		 2783 /*
2784 * Check legality
2785 */
1ce4f28b 2786
8ae55f04 2787 if (addr_len != sizeof(struct sockaddr))
1da177e4 2788 return -EINVAL;
40d4e3df 2789 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2790
3b1e0a65 2791 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2792 if (dev)
1da177e4 2793 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2794 return err;
2795}
1da177e4
LT		 2796
2797static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2798{
40d4e3df
ED		 2799 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2800 struct sock *sk = sock->sk;
1da177e4
LT		 2801 struct net_device *dev = NULL;
2802 int err;
2803
2804
2805 /*
2806 * Check legality
2807 */
1ce4f28b 2808
1da177e4
LT		 2809 if (addr_len < sizeof(struct sockaddr_ll))
2810 return -EINVAL;
2811 if (sll->sll_family != AF_PACKET)
2812 return -EINVAL;
2813
2814 if (sll->sll_ifindex) {
2815 err = -ENODEV;
3b1e0a65 2816 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2817 if (dev == NULL)
2818 goto out;
2819 }
2820 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2821
2822out:
2823 return err;
2824}
2825
2826static struct proto packet_proto = {
2827 .name = "PACKET",
2828 .owner = THIS_MODULE,
2829 .obj_size = sizeof(struct packet_sock),
2830};
2831
2832/*
1ce4f28b 2833 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2834 */
2835
3f378b68
EP		 2836static int packet_create(struct net *net, struct socket *sock, int protocol,
2837 int kern)
1da177e4
LT		 2838{
2839 struct sock *sk;
2840 struct packet_sock *po;
0e11c91e 2841 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2842 int err;
2843
df008c91 2844 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2845 return -EPERM;
be02097c
DM		 2846 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2847 sock->type != SOCK_PACKET)
1da177e4
LT		 2848 return -ESOCKTNOSUPPORT;
2849
2850 sock->state = SS_UNCONNECTED;
2851
2852 err = -ENOBUFS;
11aa9c28 2853 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
1da177e4
LT		 2854 if (sk == NULL)
2855 goto out;
2856
2857 sock->ops = &packet_ops;
1da177e4
LT		 2858 if (sock->type == SOCK_PACKET)
2859 sock->ops = &packet_ops_spkt;
be02097c 2860
1da177e4
LT		 2861 sock_init_data(sock, sk);
2862
2863 po = pkt_sk(sk);
2864 sk->sk_family = PF_PACKET;
0e11c91e 2865 po->num = proto;
d346a3fa 2866 po->xmit = dev_queue_xmit;
66e56cd4 2867
b0138408
DB		 2868 err = packet_alloc_pending(po);
2869 if (err)
2870 goto out2;
2871
66e56cd4 2872 packet_cached_dev_reset(po);
1da177e4
LT		 2873
2874 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2875 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2876
2877 /*
2878 * Attach a protocol block
2879 */
2880
2881 spin_lock_init(&po->bind_lock);
905db440 2882 mutex_init(&po->pg_vec_lock);
0648ab70 2883 po->rollover = NULL;
1da177e4 2884 po->prot_hook.func = packet_rcv;
be02097c 2885
1da177e4
LT		 2886 if (sock->type == SOCK_PACKET)
2887 po->prot_hook.func = packet_rcv_spkt;
be02097c 2888
1da177e4
LT		 2889 po->prot_hook.af_packet_priv = sk;
2890
0e11c91e
AV		 2891 if (proto) {
2892 po->prot_hook.type = proto;
ce06b03e 2893 register_prot_hook(sk);
1da177e4
LT		 2894 }
2895
0fa7fa98 2896 mutex_lock(&net->packet.sklist_lock);
808f5114 2897 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2898 mutex_unlock(&net->packet.sklist_lock);
2899
2900 preempt_disable();
3680453c 2901 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2902 preempt_enable();
808f5114 2903
40d4e3df 2904 return 0;
b0138408
DB		 2905out2:
2906 sk_free(sk);
1da177e4
LT		 2907out:
2908 return err;
2909}
2910
2911/*
2912 * Pull a packet from our receive queue and hand it to the user.
2913 * If necessary we block.
2914 */
2915
1b784140
YX		 2916static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2917 int flags)
1da177e4
LT		 2918{
2919 struct sock *sk = sock->sk;
2920 struct sk_buff *skb;
2921 int copied, err;
bfd5f4a3 2922 int vnet_hdr_len = 0;
2472d761 2923 unsigned int origlen = 0;
1da177e4
LT		 2924
2925 err = -EINVAL;
ed85b565 2926 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2927 goto out;
2928
2929#if 0
2930 /* What error should we return now? EUNATTACH? */
2931 if (pkt_sk(sk)->ifindex < 0)
2932 return -ENODEV;
2933#endif
2934
ed85b565 2935 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2936 err = sock_recv_errqueue(sk, msg, len,
2937 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2938 goto out;
2939 }
2940
1da177e4
LT		 2941 /*
2942 * Call the generic datagram receiver. This handles all sorts
2943 * of horrible races and re-entrancy so we can forget about it
2944 * in the protocol layers.
2945 *
2946 * Now it will return ENETDOWN, if device have just gone down,
2947 * but then it will block.
2948 */
2949
40d4e3df 2950 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2951
2952 /*
1ce4f28b 2953 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2954 * handles the blocking we don't see and worry about blocking
2955 * retries.
2956 */
2957
8ae55f04 2958 if (skb == NULL)
1da177e4
LT		 2959 goto out;
2960
bfd5f4a3
SS		 2961 if (pkt_sk(sk)->has_vnet_hdr) {
2962 struct virtio_net_hdr vnet_hdr = { 0 };
2963
2964 err = -EINVAL;
2965 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2966 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2967 goto out_free;
2968
1f18b717
MK		 2969 len -= vnet_hdr_len;
2970
bfd5f4a3
SS		 2971 if (skb_is_gso(skb)) {
2972 struct skb_shared_info *sinfo = skb_shinfo(skb);
2973
2974 /* This is a hint as to how much should be linear. */
dc9e5153
MT		 2975 vnet_hdr.hdr_len =
2976 __cpu_to_virtio16(false, skb_headlen(skb));
2977 vnet_hdr.gso_size =
2978 __cpu_to_virtio16(false, sinfo->gso_size);
bfd5f4a3
SS		 2979 if (sinfo->gso_type & SKB_GSO_TCPV4)
2980 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2981 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2982 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2983 else if (sinfo->gso_type & SKB_GSO_UDP)
2984 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2985 else if (sinfo->gso_type & SKB_GSO_FCOE)
2986 goto out_free;
2987 else
2988 BUG();
2989 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2990 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2991 } else
2992 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2993
2994 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2995 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
dc9e5153
MT		 2996 vnet_hdr.csum_start = __cpu_to_virtio16(false,
2997 skb_checksum_start_offset(skb));
2998 vnet_hdr.csum_offset = __cpu_to_virtio16(false,
2999 skb->csum_offset);
10a8d94a
JW		 3000 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3001 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 3002 } /* else everything is zero */
3003
7eab8d9e 3004 err = memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_len);
bfd5f4a3
SS		 3005 if (err < 0)
3006 goto out_free;
3007 }
3008
f3d33426
HFS		 3009 /* You lose any data beyond the buffer you gave. If it worries
3010 * a user program they can ask the device for its MTU
3011 * anyway.
1da177e4 3012 */
1da177e4 3013 copied = skb->len;
40d4e3df
ED		 3014 if (copied > len) {
3015 copied = len;
3016 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 3017 }
3018
51f3d02b 3019 err = skb_copy_datagram_msg(skb, 0, msg, copied);
1da177e4
LT		 3020 if (err)
3021 goto out_free;
3022
2472d761
EB		 3023 if (sock->type != SOCK_PACKET) {
3024 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3025
3026 /* Original length was stored in sockaddr_ll fields */
3027 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3028 sll->sll_family = AF_PACKET;
3029 sll->sll_protocol = skb->protocol;
3030 }
3031
3b885787 3032 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 3033
f3d33426
HFS		 3034 if (msg->msg_name) {
3035 /* If the address length field is there to be filled
3036 * in, we fill it in now.
3037 */
3038 if (sock->type == SOCK_PACKET) {
342dfc30 3039 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 3040 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3041 } else {
3042 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2472d761 3043
f3d33426
HFS		 3044 msg->msg_namelen = sll->sll_halen +
3045 offsetof(struct sockaddr_ll, sll_addr);
3046 }
ffbc6111
HX		 3047 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3048 msg->msg_namelen);
f3d33426 3049 }
1da177e4 3050
8dc41944 3051 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 3052 struct tpacket_auxdata aux;
3053
3054 aux.tp_status = TP_STATUS_USER;
3055 if (skb->ip_summed == CHECKSUM_PARTIAL)
3056 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 3057 else if (skb->pkt_type != PACKET_OUTGOING &&
3058 (skb->ip_summed == CHECKSUM_COMPLETE ||
3059 skb_csum_unnecessary(skb)))
3060 aux.tp_status |= TP_STATUS_CSUM_VALID;
3061
2472d761 3062 aux.tp_len = origlen;
ffbc6111
HX		 3063 aux.tp_snaplen = skb->len;
3064 aux.tp_mac = 0;
bbe735e4 3065 aux.tp_net = skb_network_offset(skb);
df8a39de
JP		 3066 if (skb_vlan_tag_present(skb)) {
3067 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 3068 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3069 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 3070 } else {
3071 aux.tp_vlan_tci = 0;
a0cdfcf3 3072 aux.tp_vlan_tpid = 0;
a3bcc23e 3073 }
ffbc6111 3074 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3075 }
3076
1da177e4
LT		 3077 /*
3078 * Free or return the buffer as appropriate. Again this
3079 * hides all the races and re-entrancy issues from us.
3080 */
bfd5f4a3 3081 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3082
3083out_free:
3084 skb_free_datagram(sk, skb);
3085out:
3086 return err;
3087}
3088
1da177e4
LT		 3089static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3090 int *uaddr_len, int peer)
3091{
3092 struct net_device *dev;
3093 struct sock *sk = sock->sk;
3094
3095 if (peer)
3096 return -EOPNOTSUPP;
3097
3098 uaddr->sa_family = AF_PACKET;
2dc85bf3 3099 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3100 rcu_read_lock();
3101 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3102 if (dev)
2dc85bf3 3103 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3104 rcu_read_unlock();
1da177e4
LT		 3105 *uaddr_len = sizeof(*uaddr);
3106
3107 return 0;
3108}
1da177e4
LT		 3109
3110static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3111 int *uaddr_len, int peer)
3112{
3113 struct net_device *dev;
3114 struct sock *sk = sock->sk;
3115 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3116 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3117
3118 if (peer)
3119 return -EOPNOTSUPP;
3120
3121 sll->sll_family = AF_PACKET;
3122 sll->sll_ifindex = po->ifindex;
3123 sll->sll_protocol = po->num;
67286640 3124 sll->sll_pkttype = 0;
654d1f8a
ED		 3125 rcu_read_lock();
3126 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3127 if (dev) {
3128 sll->sll_hatype = dev->type;
3129 sll->sll_halen = dev->addr_len;
3130 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3131 } else {
3132 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3133 sll->sll_halen = 0;
3134 }
654d1f8a 3135 rcu_read_unlock();
0fb375fb 3136 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3137
3138 return 0;
3139}
3140
2aeb0b88
WC		 3141static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3142 int what)
1da177e4
LT		 3143{
3144 switch (i->type) {
3145 case PACKET_MR_MULTICAST:
1162563f
JP		 3146 if (i->alen != dev->addr_len)
3147 return -EINVAL;
1da177e4 3148 if (what > 0)
22bedad3 3149 return dev_mc_add(dev, i->addr);
1da177e4 3150 else
22bedad3 3151 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3152 break;
3153 case PACKET_MR_PROMISC:
2aeb0b88 3154 return dev_set_promiscuity(dev, what);
1da177e4 3155 case PACKET_MR_ALLMULTI:
2aeb0b88 3156 return dev_set_allmulti(dev, what);
d95ed927 3157 case PACKET_MR_UNICAST:
1162563f
JP		 3158 if (i->alen != dev->addr_len)
3159 return -EINVAL;
d95ed927 3160 if (what > 0)
a748ee24 3161 return dev_uc_add(dev, i->addr);
d95ed927 3162 else
a748ee24 3163 return dev_uc_del(dev, i->addr);
d95ed927 3164 break;
40d4e3df
ED		 3165 default:
3166 break;
1da177e4 3167 }
2aeb0b88 3168 return 0;
1da177e4
LT		 3169}
3170
82f17091
FR		 3171static void packet_dev_mclist_delete(struct net_device *dev,
3172 struct packet_mclist **mlp)
1da177e4 3173{
82f17091
FR		 3174 struct packet_mclist *ml;
3175
3176 while ((ml = *mlp) != NULL) {
3177 if (ml->ifindex == dev->ifindex) {
3178 packet_dev_mc(dev, ml, -1);
3179 *mlp = ml->next;
3180 kfree(ml);
3181 } else
3182 mlp = &ml->next;
1da177e4
LT		 3183 }
3184}
3185
0fb375fb 3186static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3187{
3188 struct packet_sock *po = pkt_sk(sk);
3189 struct packet_mclist *ml, *i;
3190 struct net_device *dev;
3191 int err;
3192
3193 rtnl_lock();
3194
3195 err = -ENODEV;
3b1e0a65 3196 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3197 if (!dev)
3198 goto done;
3199
3200 err = -EINVAL;
1162563f 3201 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3202 goto done;
3203
3204 err = -ENOBUFS;
8b3a7005 3205 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3206 if (i == NULL)
3207 goto done;
3208
3209 err = 0;
3210 for (ml = po->mclist; ml; ml = ml->next) {
3211 if (ml->ifindex == mreq->mr_ifindex &&
3212 ml->type == mreq->mr_type &&
3213 ml->alen == mreq->mr_alen &&
3214 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3215 ml->count++;
3216 /* Free the new element ... */
3217 kfree(i);
3218 goto done;
3219 }
3220 }
3221
3222 i->type = mreq->mr_type;
3223 i->ifindex = mreq->mr_ifindex;
3224 i->alen = mreq->mr_alen;
3225 memcpy(i->addr, mreq->mr_address, i->alen);
3226 i->count = 1;
3227 i->next = po->mclist;
3228 po->mclist = i;
2aeb0b88
WC		 3229 err = packet_dev_mc(dev, i, 1);
3230 if (err) {
3231 po->mclist = i->next;
3232 kfree(i);
3233 }
1da177e4
LT		 3234
3235done:
3236 rtnl_unlock();
3237 return err;
3238}
3239
0fb375fb 3240static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3241{
3242 struct packet_mclist *ml, **mlp;
3243
3244 rtnl_lock();
3245
3246 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3247 if (ml->ifindex == mreq->mr_ifindex &&
3248 ml->type == mreq->mr_type &&
3249 ml->alen == mreq->mr_alen &&
3250 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3251 if (--ml->count == 0) {
3252 struct net_device *dev;
3253 *mlp = ml->next;
ad959e76
ED		 3254 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3255 if (dev)
1da177e4 3256 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3257 kfree(ml);
3258 }
82f17091 3259 break;
1da177e4
LT		 3260 }
3261 }
3262 rtnl_unlock();
82f17091 3263 return 0;
1da177e4
LT		 3264}
3265
3266static void packet_flush_mclist(struct sock *sk)
3267{
3268 struct packet_sock *po = pkt_sk(sk);
3269 struct packet_mclist *ml;
3270
3271 if (!po->mclist)
3272 return;
3273
3274 rtnl_lock();
3275 while ((ml = po->mclist) != NULL) {
3276 struct net_device *dev;
3277
3278 po->mclist = ml->next;
ad959e76
ED		 3279 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3280 if (dev != NULL)
1da177e4 3281 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3282 kfree(ml);
3283 }
3284 rtnl_unlock();
3285}
1da177e4
LT		 3286
3287static int
b7058842 3288packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3289{
3290 struct sock *sk = sock->sk;
8dc41944 3291 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3292 int ret;
3293
3294 if (level != SOL_PACKET)
3295 return -ENOPROTOOPT;
3296
69e3c75f 3297 switch (optname) {
1ce4f28b 3298 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3299 case PACKET_DROP_MEMBERSHIP:
3300 {
0fb375fb
EB		 3301 struct packet_mreq_max mreq;
3302 int len = optlen;
3303 memset(&mreq, 0, sizeof(mreq));
3304 if (len < sizeof(struct packet_mreq))
1da177e4 3305 return -EINVAL;
0fb375fb
EB		 3306 if (len > sizeof(mreq))
3307 len = sizeof(mreq);
40d4e3df 3308 if (copy_from_user(&mreq, optval, len))
1da177e4 3309 return -EFAULT;
0fb375fb
EB		 3310 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3311 return -EINVAL;
1da177e4
LT		 3312 if (optname == PACKET_ADD_MEMBERSHIP)
3313 ret = packet_mc_add(sk, &mreq);
3314 else
3315 ret = packet_mc_drop(sk, &mreq);
3316 return ret;
3317 }
a2efcfa0 3318
1da177e4 3319 case PACKET_RX_RING:
69e3c75f 3320 case PACKET_TX_RING:
1da177e4 3321 {
f6fb8f10 3322 union tpacket_req_u req_u;
3323 int len;
1da177e4 3324
f6fb8f10 3325 switch (po->tp_version) {
3326 case TPACKET_V1:
3327 case TPACKET_V2:
3328 len = sizeof(req_u.req);
3329 break;
3330 case TPACKET_V3:
3331 default:
3332 len = sizeof(req_u.req3);
3333 break;
3334 }
3335 if (optlen < len)
1da177e4 3336 return -EINVAL;
bfd5f4a3
SS		 3337 if (pkt_sk(sk)->has_vnet_hdr)
3338 return -EINVAL;
f6fb8f10 3339 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3340 return -EFAULT;
f6fb8f10 3341 return packet_set_ring(sk, &req_u, 0,
3342 optname == PACKET_TX_RING);
1da177e4
LT		 3343 }
3344 case PACKET_COPY_THRESH:
3345 {
3346 int val;
3347
40d4e3df 3348 if (optlen != sizeof(val))
1da177e4 3349 return -EINVAL;
40d4e3df 3350 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3351 return -EFAULT;
3352
3353 pkt_sk(sk)->copy_thresh = val;
3354 return 0;
3355 }
bbd6ef87
PM		 3356 case PACKET_VERSION:
3357 {
3358 int val;
3359
3360 if (optlen != sizeof(val))
3361 return -EINVAL;
69e3c75f 3362 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3363 return -EBUSY;
3364 if (copy_from_user(&val, optval, sizeof(val)))
3365 return -EFAULT;
3366 switch (val) {
3367 case TPACKET_V1:
3368 case TPACKET_V2:
f6fb8f10 3369 case TPACKET_V3:
bbd6ef87
PM		 3370 po->tp_version = val;
3371 return 0;
3372 default:
3373 return -EINVAL;
3374 }
3375 }
8913336a
PM		 3376 case PACKET_RESERVE:
3377 {
3378 unsigned int val;
3379
3380 if (optlen != sizeof(val))
3381 return -EINVAL;
69e3c75f 3382 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3383 return -EBUSY;
3384 if (copy_from_user(&val, optval, sizeof(val)))
3385 return -EFAULT;
3386 po->tp_reserve = val;
3387 return 0;
3388 }
69e3c75f
JB		 3389 case PACKET_LOSS:
3390 {
3391 unsigned int val;
3392
3393 if (optlen != sizeof(val))
3394 return -EINVAL;
3395 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3396 return -EBUSY;
3397 if (copy_from_user(&val, optval, sizeof(val)))
3398 return -EFAULT;
3399 po->tp_loss = !!val;
3400 return 0;
3401 }
8dc41944
HX		 3402 case PACKET_AUXDATA:
3403 {
3404 int val;
3405
3406 if (optlen < sizeof(val))
3407 return -EINVAL;
3408 if (copy_from_user(&val, optval, sizeof(val)))
3409 return -EFAULT;
3410
3411 po->auxdata = !!val;
3412 return 0;
3413 }
80feaacb
PWJ		 3414 case PACKET_ORIGDEV:
3415 {
3416 int val;
3417
3418 if (optlen < sizeof(val))
3419 return -EINVAL;
3420 if (copy_from_user(&val, optval, sizeof(val)))
3421 return -EFAULT;
3422
3423 po->origdev = !!val;
3424 return 0;
3425 }
bfd5f4a3
SS		 3426 case PACKET_VNET_HDR:
3427 {
3428 int val;
3429
3430 if (sock->type != SOCK_RAW)
3431 return -EINVAL;
3432 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3433 return -EBUSY;
3434 if (optlen < sizeof(val))
3435 return -EINVAL;
3436 if (copy_from_user(&val, optval, sizeof(val)))
3437 return -EFAULT;
3438
3439 po->has_vnet_hdr = !!val;
3440 return 0;
3441 }
614f60fa
SM		 3442 case PACKET_TIMESTAMP:
3443 {
3444 int val;
3445
3446 if (optlen != sizeof(val))
3447 return -EINVAL;
3448 if (copy_from_user(&val, optval, sizeof(val)))
3449 return -EFAULT;
3450
3451 po->tp_tstamp = val;
3452 return 0;
3453 }
dc99f600
DM		 3454 case PACKET_FANOUT:
3455 {
3456 int val;
3457
3458 if (optlen != sizeof(val))
3459 return -EINVAL;
3460 if (copy_from_user(&val, optval, sizeof(val)))
3461 return -EFAULT;
3462
3463 return fanout_add(sk, val & 0xffff, val >> 16);
3464 }
5920cd3a
PC		 3465 case PACKET_TX_HAS_OFF:
3466 {
3467 unsigned int val;
3468
3469 if (optlen != sizeof(val))
3470 return -EINVAL;
3471 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3472 return -EBUSY;
3473 if (copy_from_user(&val, optval, sizeof(val)))
3474 return -EFAULT;
3475 po->tp_tx_has_off = !!val;
3476 return 0;
3477 }
d346a3fa
DB		 3478 case PACKET_QDISC_BYPASS:
3479 {
3480 int val;
3481
3482 if (optlen != sizeof(val))
3483 return -EINVAL;
3484 if (copy_from_user(&val, optval, sizeof(val)))
3485 return -EFAULT;
3486
3487 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3488 return 0;
3489 }
1da177e4
LT		 3490 default:
3491 return -ENOPROTOOPT;
3492 }
3493}
3494
3495static int packet_getsockopt(struct socket *sock, int level, int optname,
3496 char __user *optval, int __user *optlen)
3497{
3498 int len;
c06fff6e 3499 int val, lv = sizeof(val);
1da177e4
LT		 3500 struct sock *sk = sock->sk;
3501 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3502 void *data = &val;
ee80fbf3 3503 union tpacket_stats_u st;
1da177e4
LT		 3504
3505 if (level != SOL_PACKET)
3506 return -ENOPROTOOPT;
3507
8ae55f04
KK		 3508 if (get_user(len, optlen))
3509 return -EFAULT;
1da177e4
LT		 3510
3511 if (len < 0)
3512 return -EINVAL;
1ce4f28b 3513
69e3c75f 3514 switch (optname) {
1da177e4 3515 case PACKET_STATISTICS:
1da177e4 3516 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3517 memcpy(&st, &po->stats, sizeof(st));
3518 memset(&po->stats, 0, sizeof(po->stats));
3519 spin_unlock_bh(&sk->sk_receive_queue.lock);
3520
f6fb8f10 3521 if (po->tp_version == TPACKET_V3) {
c06fff6e 3522 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3523 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3524 data = &st.stats3;
f6fb8f10 3525 } else {
c06fff6e 3526 lv = sizeof(struct tpacket_stats);
8bcdeaff 3527 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3528 data = &st.stats1;
f6fb8f10 3529 }
ee80fbf3 3530
8dc41944
HX		 3531 break;
3532 case PACKET_AUXDATA:
8dc41944 3533 val = po->auxdata;
80feaacb
PWJ		 3534 break;
3535 case PACKET_ORIGDEV:
80feaacb 3536 val = po->origdev;
bfd5f4a3
SS		 3537 break;
3538 case PACKET_VNET_HDR:
bfd5f4a3 3539 val = po->has_vnet_hdr;
1da177e4 3540 break;
bbd6ef87 3541 case PACKET_VERSION:
bbd6ef87 3542 val = po->tp_version;
bbd6ef87
PM		 3543 break;
3544 case PACKET_HDRLEN:
3545 if (len > sizeof(int))
3546 len = sizeof(int);
3547 if (copy_from_user(&val, optval, len))
3548 return -EFAULT;
3549 switch (val) {
3550 case TPACKET_V1:
3551 val = sizeof(struct tpacket_hdr);
3552 break;
3553 case TPACKET_V2:
3554 val = sizeof(struct tpacket2_hdr);
3555 break;
f6fb8f10 3556 case TPACKET_V3:
3557 val = sizeof(struct tpacket3_hdr);
3558 break;
bbd6ef87
PM		 3559 default:
3560 return -EINVAL;
3561 }
bbd6ef87 3562 break;
8913336a 3563 case PACKET_RESERVE:
8913336a 3564 val = po->tp_reserve;
8913336a 3565 break;
69e3c75f 3566 case PACKET_LOSS:
69e3c75f 3567 val = po->tp_loss;
69e3c75f 3568 break;
614f60fa 3569 case PACKET_TIMESTAMP:
614f60fa 3570 val = po->tp_tstamp;
614f60fa 3571 break;
dc99f600 3572 case PACKET_FANOUT:
dc99f600
DM		 3573 val = (po->fanout ?
3574 ((u32)po->fanout->id |
77f65ebd
WB		 3575 ((u32)po->fanout->type << 16) |
3576 ((u32)po->fanout->flags << 24)) :
dc99f600 3577 0);
dc99f600 3578 break;
5920cd3a
PC		 3579 case PACKET_TX_HAS_OFF:
3580 val = po->tp_tx_has_off;
3581 break;
d346a3fa
DB		 3582 case PACKET_QDISC_BYPASS:
3583 val = packet_use_direct_xmit(po);
3584 break;
1da177e4
LT		 3585 default:
3586 return -ENOPROTOOPT;
3587 }
3588
c06fff6e
ED		 3589 if (len > lv)
3590 len = lv;
8ae55f04
KK		 3591 if (put_user(len, optlen))
3592 return -EFAULT;
8dc41944
HX		 3593 if (copy_to_user(optval, data, len))
3594 return -EFAULT;
8ae55f04 3595 return 0;
1da177e4
LT		 3596}
3597
3598
351638e7
JP		 3599static int packet_notifier(struct notifier_block *this,
3600 unsigned long msg, void *ptr)
1da177e4
LT		 3601{
3602 struct sock *sk;
351638e7 3603 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3604 struct net *net = dev_net(dev);
1da177e4 3605
808f5114 3606 rcu_read_lock();
b67bfe0d 3607 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3608 struct packet_sock *po = pkt_sk(sk);
3609
3610 switch (msg) {
3611 case NETDEV_UNREGISTER:
1da177e4 3612 if (po->mclist)
82f17091 3613 packet_dev_mclist_delete(dev, &po->mclist);
a2efcfa0
DM		 3614 /* fallthrough */
3615
1da177e4
LT		 3616 case NETDEV_DOWN:
3617 if (dev->ifindex == po->ifindex) {
3618 spin_lock(&po->bind_lock);
3619 if (po->running) {
ce06b03e 3620 __unregister_prot_hook(sk, false);
1da177e4
LT		 3621 sk->sk_err = ENETDOWN;
3622 if (!sock_flag(sk, SOCK_DEAD))
3623 sk->sk_error_report(sk);
3624 }
3625 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3626 packet_cached_dev_reset(po);
1da177e4 3627 po->ifindex = -1;
160ff18a
BG		 3628 if (po->prot_hook.dev)
3629 dev_put(po->prot_hook.dev);
1da177e4
LT		 3630 po->prot_hook.dev = NULL;
3631 }
3632 spin_unlock(&po->bind_lock);
3633 }
3634 break;
3635 case NETDEV_UP:
808f5114 3636 if (dev->ifindex == po->ifindex) {
3637 spin_lock(&po->bind_lock);
ce06b03e
DM		 3638 if (po->num)
3639 register_prot_hook(sk);
808f5114 3640 spin_unlock(&po->bind_lock);
1da177e4 3641 }
1da177e4
LT		 3642 break;
3643 }
3644 }
808f5114 3645 rcu_read_unlock();
1da177e4
LT		 3646 return NOTIFY_DONE;
3647}
3648
3649
3650static int packet_ioctl(struct socket *sock, unsigned int cmd,
3651 unsigned long arg)
3652{
3653 struct sock *sk = sock->sk;
3654
69e3c75f 3655 switch (cmd) {
40d4e3df
ED		 3656 case SIOCOUTQ:
3657 {
3658 int amount = sk_wmem_alloc_get(sk);
31e6d363 3659
40d4e3df
ED		 3660 return put_user(amount, (int __user *)arg);
3661 }
3662 case SIOCINQ:
3663 {
3664 struct sk_buff *skb;
3665 int amount = 0;
3666
3667 spin_lock_bh(&sk->sk_receive_queue.lock);
3668 skb = skb_peek(&sk->sk_receive_queue);
3669 if (skb)
3670 amount = skb->len;
3671 spin_unlock_bh(&sk->sk_receive_queue.lock);
3672 return put_user(amount, (int __user *)arg);
3673 }
3674 case SIOCGSTAMP:
3675 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3676 case SIOCGSTAMPNS:
3677 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3678
1da177e4 3679#ifdef CONFIG_INET
40d4e3df
ED		 3680 case SIOCADDRT:
3681 case SIOCDELRT:
3682 case SIOCDARP:
3683 case SIOCGARP:
3684 case SIOCSARP:
3685 case SIOCGIFADDR:
3686 case SIOCSIFADDR:
3687 case SIOCGIFBRDADDR:
3688 case SIOCSIFBRDADDR:
3689 case SIOCGIFNETMASK:
3690 case SIOCSIFNETMASK:
3691 case SIOCGIFDSTADDR:
3692 case SIOCSIFDSTADDR:
3693 case SIOCSIFFLAGS:
40d4e3df 3694 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3695#endif
3696
40d4e3df
ED		 3697 default:
3698 return -ENOIOCTLCMD;
1da177e4
LT		 3699 }
3700 return 0;
3701}
3702
40d4e3df 3703static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3704 poll_table *wait)
3705{
3706 struct sock *sk = sock->sk;
3707 struct packet_sock *po = pkt_sk(sk);
3708 unsigned int mask = datagram_poll(file, sock, wait);
3709
3710 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3711 if (po->rx_ring.pg_vec) {
f6fb8f10 3712 if (!packet_previous_rx_frame(po, &po->rx_ring,
3713 TP_STATUS_KERNEL))
1da177e4
LT		 3714 mask |= POLLIN | POLLRDNORM;
3715 }
3716 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3717 spin_lock_bh(&sk->sk_write_queue.lock);
3718 if (po->tx_ring.pg_vec) {
3719 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3720 mask |= POLLOUT | POLLWRNORM;
3721 }
3722 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3723 return mask;
3724}
3725
3726
3727/* Dirty? Well, I still did not learn better way to account
3728 * for user mmaps.
3729 */
3730
3731static void packet_mm_open(struct vm_area_struct *vma)
3732{
3733 struct file *file = vma->vm_file;
40d4e3df 3734 struct socket *sock = file->private_data;
1da177e4 3735 struct sock *sk = sock->sk;
1ce4f28b 3736
1da177e4
LT		 3737 if (sk)
3738 atomic_inc(&pkt_sk(sk)->mapped);
3739}
3740
3741static void packet_mm_close(struct vm_area_struct *vma)
3742{
3743 struct file *file = vma->vm_file;
40d4e3df 3744 struct socket *sock = file->private_data;
1da177e4 3745 struct sock *sk = sock->sk;
1ce4f28b 3746
1da177e4
LT		 3747 if (sk)
3748 atomic_dec(&pkt_sk(sk)->mapped);
3749}
3750
f0f37e2f 3751static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3752 .open = packet_mm_open,
3753 .close = packet_mm_close,
1da177e4
LT		 3754};
3755
0e3125c7
NH		 3756static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3757 unsigned int len)
1da177e4
LT		 3758{
3759 int i;
3760
4ebf0ae2 3761 for (i = 0; i < len; i++) {
0e3125c7 3762 if (likely(pg_vec[i].buffer)) {
c56b4d90 3763 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3764 vfree(pg_vec[i].buffer);
3765 else
3766 free_pages((unsigned long)pg_vec[i].buffer,
3767 order);
3768 pg_vec[i].buffer = NULL;
3769 }
1da177e4
LT		 3770 }
3771 kfree(pg_vec);
3772}
3773
eea49cc9 3774static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3775{
f0d4eb29 3776 char *buffer;
0e3125c7
NH		 3777 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3778 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3779
3780 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3781 if (buffer)
3782 return buffer;
3783
f0d4eb29 3784 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3785 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3786 if (buffer)
3787 return buffer;
3788
f0d4eb29 3789 /* vmalloc failed, lets dig into swap here */
0e3125c7 3790 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3791 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3792 if (buffer)
3793 return buffer;
3794
f0d4eb29 3795 /* complete and utter failure */
0e3125c7 3796 return NULL;
4ebf0ae2
DM		 3797}
3798
0e3125c7 3799static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3800{
3801 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3802 struct pgv *pg_vec;
4ebf0ae2
DM		 3803 int i;
3804
0e3125c7 3805 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3806 if (unlikely(!pg_vec))
3807 goto out;
3808
3809 for (i = 0; i < block_nr; i++) {
c56b4d90 3810 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3811 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3812 goto out_free_pgvec;
3813 }
3814
3815out:
3816 return pg_vec;
3817
3818out_free_pgvec:
3819 free_pg_vec(pg_vec, order, block_nr);
3820 pg_vec = NULL;
3821 goto out;
3822}
1da177e4 3823
f6fb8f10 3824static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3825 int closing, int tx_ring)
1da177e4 3826{
0e3125c7 3827 struct pgv *pg_vec = NULL;
1da177e4 3828 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3829 int was_running, order = 0;
69e3c75f
JB		 3830 struct packet_ring_buffer *rb;
3831 struct sk_buff_head *rb_queue;
0e11c91e 3832 __be16 num;
f6fb8f10 3833 int err = -EINVAL;
3834 /* Added to avoid minimal code churn */
3835 struct tpacket_req *req = &req_u->req;
3836
3837 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3838 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3839 WARN(1, "Tx-ring is not supported.\n");
3840 goto out;
3841 }
1ce4f28b 3842
69e3c75f
JB		 3843 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3844 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3845
69e3c75f
JB		 3846 err = -EBUSY;
3847 if (!closing) {
3848 if (atomic_read(&po->mapped))
3849 goto out;
b0138408 3850 if (packet_read_pending(rb))
69e3c75f
JB		 3851 goto out;
3852 }
1da177e4 3853
69e3c75f
JB		 3854 if (req->tp_block_nr) {
3855 /* Sanity tests and some calculations */
3856 err = -EBUSY;
3857 if (unlikely(rb->pg_vec))
3858 goto out;
1da177e4 3859
bbd6ef87
PM		 3860 switch (po->tp_version) {
3861 case TPACKET_V1:
3862 po->tp_hdrlen = TPACKET_HDRLEN;
3863 break;
3864 case TPACKET_V2:
3865 po->tp_hdrlen = TPACKET2_HDRLEN;
3866 break;
f6fb8f10 3867 case TPACKET_V3:
3868 po->tp_hdrlen = TPACKET3_HDRLEN;
3869 break;
bbd6ef87
PM		 3870 }
3871
69e3c75f 3872 err = -EINVAL;
4ebf0ae2 3873 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3874 goto out;
4ebf0ae2 3875 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3876 goto out;
dc808110
ED		 3877 if (po->tp_version >= TPACKET_V3 &&
3878 (int)(req->tp_block_size -
3879 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3880 goto out;
8913336a 3881 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3882 po->tp_reserve))
3883 goto out;
4ebf0ae2 3884 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3885 goto out;
1da177e4 3886
69e3c75f
JB		 3887 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3888 if (unlikely(rb->frames_per_block <= 0))
3889 goto out;
3890 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3891 req->tp_frame_nr))
3892 goto out;
1da177e4
LT		 3893
3894 err = -ENOMEM;
4ebf0ae2
DM		 3895 order = get_order(req->tp_block_size);
3896 pg_vec = alloc_pg_vec(req, order);
3897 if (unlikely(!pg_vec))
1da177e4 3898 goto out;
f6fb8f10 3899 switch (po->tp_version) {
3900 case TPACKET_V3:
3901 /* Transmit path is not supported. We checked
3902 * it above but just being paranoid
3903 */
3904 if (!tx_ring)
3905 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 3906 break;
f6fb8f10 3907 default:
3908 break;
3909 }
69e3c75f
JB		 3910 }
3911 /* Done */
3912 else {
3913 err = -EINVAL;
4ebf0ae2 3914 if (unlikely(req->tp_frame_nr))
69e3c75f 3915 goto out;
1da177e4
LT		 3916 }
3917
3918 lock_sock(sk);
3919
3920 /* Detach socket from network */
3921 spin_lock(&po->bind_lock);
3922 was_running = po->running;
3923 num = po->num;
3924 if (was_running) {
1da177e4 3925 po->num = 0;
ce06b03e 3926 __unregister_prot_hook(sk, false);
1da177e4
LT		 3927 }
3928 spin_unlock(&po->bind_lock);
1ce4f28b 3929
1da177e4
LT		 3930 synchronize_net();
3931
3932 err = -EBUSY;
905db440 3933 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3934 if (closing || atomic_read(&po->mapped) == 0) {
3935 err = 0;
69e3c75f 3936 spin_lock_bh(&rb_queue->lock);
c053fd96 3937 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3938 rb->frame_max = (req->tp_frame_nr - 1);
3939 rb->head = 0;
3940 rb->frame_size = req->tp_frame_size;
3941 spin_unlock_bh(&rb_queue->lock);
3942
c053fd96
CG		 3943 swap(rb->pg_vec_order, order);
3944 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3945
3946 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3947 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3948 tpacket_rcv : packet_rcv;
3949 skb_queue_purge(rb_queue);
1da177e4 3950 if (atomic_read(&po->mapped))
40d4e3df
ED		 3951 pr_err("packet_mmap: vma is busy: %d\n",
3952 atomic_read(&po->mapped));
1da177e4 3953 }
905db440 3954 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3955
3956 spin_lock(&po->bind_lock);
ce06b03e 3957 if (was_running) {
1da177e4 3958 po->num = num;
ce06b03e 3959 register_prot_hook(sk);
1da177e4
LT		 3960 }
3961 spin_unlock(&po->bind_lock);
f6fb8f10 3962 if (closing && (po->tp_version > TPACKET_V2)) {
3963 /* Because we don't support block-based V3 on tx-ring */
3964 if (!tx_ring)
3965 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3966 }
1da177e4
LT		 3967 release_sock(sk);
3968
1da177e4
LT		 3969 if (pg_vec)
3970 free_pg_vec(pg_vec, order, req->tp_block_nr);
3971out:
3972 return err;
3973}
3974
69e3c75f
JB		 3975static int packet_mmap(struct file *file, struct socket *sock,
3976 struct vm_area_struct *vma)
1da177e4
LT		 3977{
3978 struct sock *sk = sock->sk;
3979 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3980 unsigned long size, expected_size;
3981 struct packet_ring_buffer *rb;
1da177e4
LT		 3982 unsigned long start;
3983 int err = -EINVAL;
3984 int i;
3985
3986 if (vma->vm_pgoff)
3987 return -EINVAL;
3988
905db440 3989 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3990
3991 expected_size = 0;
3992 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3993 if (rb->pg_vec) {
3994 expected_size += rb->pg_vec_len
3995 * rb->pg_vec_pages
3996 * PAGE_SIZE;
3997 }
3998 }
3999
4000 if (expected_size == 0)
1da177e4 4001 goto out;
69e3c75f
JB		 4002
4003 size = vma->vm_end - vma->vm_start;
4004 if (size != expected_size)
1da177e4
LT		 4005 goto out;
4006
1da177e4 4007 start = vma->vm_start;
69e3c75f
JB		 4008 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4009 if (rb->pg_vec == NULL)
4010 continue;
4011
4012 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 4013 struct page *page;
4014 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 4015 int pg_num;
4016
c56b4d90
CG		 4017 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4018 page = pgv_to_page(kaddr);
69e3c75f
JB		 4019 err = vm_insert_page(vma, start, page);
4020 if (unlikely(err))
4021 goto out;
4022 start += PAGE_SIZE;
0e3125c7 4023 kaddr += PAGE_SIZE;
69e3c75f 4024 }
4ebf0ae2 4025 }
1da177e4 4026 }
69e3c75f 4027
4ebf0ae2 4028 atomic_inc(&po->mapped);
1da177e4
LT		 4029 vma->vm_ops = &packet_mmap_ops;
4030 err = 0;
4031
4032out:
905db440 4033 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4034 return err;
4035}
1da177e4 4036
90ddc4f0 4037static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 4038 .family = PF_PACKET,
4039 .owner = THIS_MODULE,
4040 .release = packet_release,
4041 .bind = packet_bind_spkt,
4042 .connect = sock_no_connect,
4043 .socketpair = sock_no_socketpair,
4044 .accept = sock_no_accept,
4045 .getname = packet_getname_spkt,
4046 .poll = datagram_poll,
4047 .ioctl = packet_ioctl,
4048 .listen = sock_no_listen,
4049 .shutdown = sock_no_shutdown,
4050 .setsockopt = sock_no_setsockopt,
4051 .getsockopt = sock_no_getsockopt,
4052 .sendmsg = packet_sendmsg_spkt,
4053 .recvmsg = packet_recvmsg,
4054 .mmap = sock_no_mmap,
4055 .sendpage = sock_no_sendpage,
4056};
1da177e4 4057
90ddc4f0 4058static const struct proto_ops packet_ops = {
1da177e4
LT		 4059 .family = PF_PACKET,
4060 .owner = THIS_MODULE,
4061 .release = packet_release,
4062 .bind = packet_bind,
4063 .connect = sock_no_connect,
4064 .socketpair = sock_no_socketpair,
4065 .accept = sock_no_accept,
1ce4f28b 4066 .getname = packet_getname,
1da177e4
LT		 4067 .poll = packet_poll,
4068 .ioctl = packet_ioctl,
4069 .listen = sock_no_listen,
4070 .shutdown = sock_no_shutdown,
4071 .setsockopt = packet_setsockopt,
4072 .getsockopt = packet_getsockopt,
4073 .sendmsg = packet_sendmsg,
4074 .recvmsg = packet_recvmsg,
4075 .mmap = packet_mmap,
4076 .sendpage = sock_no_sendpage,
4077};
4078
ec1b4cf7 4079static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4080 .family = PF_PACKET,
4081 .create = packet_create,
4082 .owner = THIS_MODULE,
4083};
4084
4085static struct notifier_block packet_netdev_notifier = {
40d4e3df 4086 .notifier_call = packet_notifier,
1da177e4
LT		 4087};
4088
4089#ifdef CONFIG_PROC_FS
1da177e4
LT		 4090
4091static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4092 __acquires(RCU)
1da177e4 4093{
e372c414 4094 struct net *net = seq_file_net(seq);
808f5114 4095
4096 rcu_read_lock();
4097 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4098}
4099
4100static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4101{
1bf40954 4102 struct net *net = seq_file_net(seq);
808f5114 4103 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4104}
4105
4106static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4107 __releases(RCU)
1da177e4 4108{
808f5114 4109 rcu_read_unlock();
1da177e4
LT		 4110}
4111
1ce4f28b 4112static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4113{
4114 if (v == SEQ_START_TOKEN)
4115 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4116 else {
b7ceabd9 4117 struct sock *s = sk_entry(v);
1da177e4
LT		 4118 const struct packet_sock *po = pkt_sk(s);
4119
4120 seq_printf(seq,
71338aa7 4121 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4122 s,
4123 atomic_read(&s->sk_refcnt),
4124 s->sk_type,
4125 ntohs(po->num),
4126 po->ifindex,
4127 po->running,
4128 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4129 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4130 sock_i_ino(s));
1da177e4
LT		 4131 }
4132
4133 return 0;
4134}
4135
56b3d975 4136static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4137 .start = packet_seq_start,
4138 .next = packet_seq_next,
4139 .stop = packet_seq_stop,
4140 .show = packet_seq_show,
4141};
4142
4143static int packet_seq_open(struct inode *inode, struct file *file)
4144{
e372c414
DL		 4145 return seq_open_net(inode, file, &packet_seq_ops,
4146 sizeof(struct seq_net_private));
1da177e4
LT		 4147}
4148
da7071d7 4149static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4150 .owner = THIS_MODULE,
4151 .open = packet_seq_open,
4152 .read = seq_read,
4153 .llseek = seq_lseek,
e372c414 4154 .release = seq_release_net,
1da177e4
LT		 4155};
4156
4157#endif
4158
2c8c1e72 4159static int __net_init packet_net_init(struct net *net)
d12d01d6 4160{
0fa7fa98 4161 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4162 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4163
d4beaa66 4164 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4165 return -ENOMEM;
4166
4167 return 0;
4168}
4169
2c8c1e72 4170static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4171{
ece31ffd 4172 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4173}
4174
4175static struct pernet_operations packet_net_ops = {
4176 .init = packet_net_init,
4177 .exit = packet_net_exit,
4178};
4179
4180
1da177e4
LT		 4181static void __exit packet_exit(void)
4182{
1da177e4 4183 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4184 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4185 sock_unregister(PF_PACKET);
4186 proto_unregister(&packet_proto);
4187}
4188
4189static int __init packet_init(void)
4190{
4191 int rc = proto_register(&packet_proto, 0);
4192
4193 if (rc != 0)
4194 goto out;
4195
4196 sock_register(&packet_family_ops);
d12d01d6 4197 register_pernet_subsys(&packet_net_ops);
1da177e4 4198 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4199out:
4200 return rc;
4201}
4202
4203module_init(packet_init);
4204module_exit(packet_exit);
4205MODULE_LICENSE("GPL");
4206MODULE_ALIAS_NETPROTO(PF_PACKET);
Ubuntu Hirsute Linux kernel mirror