]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - include/linux/skbuff.h
net: Preserve CHECKSUM_COMPLETE at validation
[mirror_ubuntu-bionic-kernel.git] / include / linux / skbuff.h
CommitLineData
1da177e4
LT
1/*
2 * Definitions for the 'struct sk_buff' memory handlers.
3 *
4 * Authors:
5 * Alan Cox, <gw4pts@gw4pts.ampr.org>
6 * Florian La Roche, <rzsfl@rz.uni-sb.de>
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
12 */
13
14#ifndef _LINUX_SKBUFF_H
15#define _LINUX_SKBUFF_H
16
1da177e4 17#include <linux/kernel.h>
fe55f6d5 18#include <linux/kmemcheck.h>
1da177e4
LT
19#include <linux/compiler.h>
20#include <linux/time.h>
187f1882 21#include <linux/bug.h>
1da177e4
LT
22#include <linux/cache.h>
23
60063497 24#include <linux/atomic.h>
1da177e4
LT
25#include <asm/types.h>
26#include <linux/spinlock.h>
1da177e4 27#include <linux/net.h>
3fc7e8a6 28#include <linux/textsearch.h>
1da177e4 29#include <net/checksum.h>
a80958f4 30#include <linux/rcupdate.h>
97fc2f08 31#include <linux/dmaengine.h>
b7aa0bf7 32#include <linux/hrtimer.h>
131ea667 33#include <linux/dma-mapping.h>
c8f44aff 34#include <linux/netdev_features.h>
363ec392 35#include <linux/sched.h>
5203cd28 36#include <net/flow_keys.h>
1da177e4 37
78ea85f1
DB
38/* A. Checksumming of received packets by device.
39 *
40 * CHECKSUM_NONE:
41 *
42 * Device failed to checksum this packet e.g. due to lack of capabilities.
43 * The packet contains full (though not verified) checksum in packet but
44 * not in skb->csum. Thus, skb->csum is undefined in this case.
45 *
46 * CHECKSUM_UNNECESSARY:
47 *
48 * The hardware you're dealing with doesn't calculate the full checksum
49 * (as in CHECKSUM_COMPLETE), but it does parse headers and verify checksums
50 * for specific protocols e.g. TCP/UDP/SCTP, then, for such packets it will
51 * set CHECKSUM_UNNECESSARY if their checksums are okay. skb->csum is still
52 * undefined in this case though. It is a bad option, but, unfortunately,
53 * nowadays most vendors do this. Apparently with the secret goal to sell
54 * you new devices, when you will add new protocol to your host, f.e. IPv6 8)
55 *
56 * CHECKSUM_COMPLETE:
57 *
58 * This is the most generic way. The device supplied checksum of the _whole_
59 * packet as seen by netif_rx() and fills out in skb->csum. Meaning, the
60 * hardware doesn't need to parse L3/L4 headers to implement this.
61 *
62 * Note: Even if device supports only some protocols, but is able to produce
63 * skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
64 *
65 * CHECKSUM_PARTIAL:
66 *
67 * This is identical to the case for output below. This may occur on a packet
68 * received directly from another Linux OS, e.g., a virtualized Linux kernel
69 * on the same host. The packet can be treated in the same way as
70 * CHECKSUM_UNNECESSARY, except that on output (i.e., forwarding) the
71 * checksum must be filled in by the OS or the hardware.
72 *
73 * B. Checksumming on output.
74 *
75 * CHECKSUM_NONE:
76 *
77 * The skb was already checksummed by the protocol, or a checksum is not
78 * required.
79 *
80 * CHECKSUM_PARTIAL:
81 *
82 * The device is required to checksum the packet as seen by hard_start_xmit()
83 * from skb->csum_start up to the end, and to record/write the checksum at
84 * offset skb->csum_start + skb->csum_offset.
85 *
86 * The device must show its capabilities in dev->features, set up at device
87 * setup time, e.g. netdev_features.h:
88 *
89 * NETIF_F_HW_CSUM - It's a clever device, it's able to checksum everything.
90 * NETIF_F_IP_CSUM - Device is dumb, it's able to checksum only TCP/UDP over
91 * IPv4. Sigh. Vendors like this way for an unknown reason.
92 * Though, see comment above about CHECKSUM_UNNECESSARY. 8)
93 * NETIF_F_IPV6_CSUM - About as dumb as the last one but does IPv6 instead.
94 * NETIF_F_... - Well, you get the picture.
95 *
96 * CHECKSUM_UNNECESSARY:
97 *
98 * Normally, the device will do per protocol specific checksumming. Protocol
99 * implementations that do not want the NIC to perform the checksum
100 * calculation should use this flag in their outgoing skbs.
101 *
102 * NETIF_F_FCOE_CRC - This indicates that the device can do FCoE FC CRC
103 * offload. Correspondingly, the FCoE protocol driver
104 * stack should use CHECKSUM_UNNECESSARY.
105 *
106 * Any questions? No questions, good. --ANK
107 */
108
60476372 109/* Don't change this without changing skb_csum_unnecessary! */
78ea85f1
DB
110#define CHECKSUM_NONE 0
111#define CHECKSUM_UNNECESSARY 1
112#define CHECKSUM_COMPLETE 2
113#define CHECKSUM_PARTIAL 3
1da177e4
LT
114
115#define SKB_DATA_ALIGN(X) (((X) + (SMP_CACHE_BYTES - 1)) & \
116 ~(SMP_CACHE_BYTES - 1))
fc910a27 117#define SKB_WITH_OVERHEAD(X) \
deea84b0 118 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
fc910a27
DM
119#define SKB_MAX_ORDER(X, ORDER) \
120 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
1da177e4
LT
121#define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0))
122#define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2))
123
87fb4b7b
ED
124/* return minimum truesize of one skb containing X bytes of data */
125#define SKB_TRUESIZE(X) ((X) + \
126 SKB_DATA_ALIGN(sizeof(struct sk_buff)) + \
127 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
128
1da177e4 129struct net_device;
716ea3a7 130struct scatterlist;
9c55e01c 131struct pipe_inode_info;
1da177e4 132
5f79e0f9 133#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
1da177e4
LT
134struct nf_conntrack {
135 atomic_t use;
1da177e4 136};
5f79e0f9 137#endif
1da177e4
LT
138
139#ifdef CONFIG_BRIDGE_NETFILTER
140struct nf_bridge_info {
bf1ac5ca
ED
141 atomic_t use;
142 unsigned int mask;
143 struct net_device *physindev;
144 struct net_device *physoutdev;
145 unsigned long data[32 / sizeof(unsigned long)];
1da177e4
LT
146};
147#endif
148
1da177e4
LT
149struct sk_buff_head {
150 /* These two members must be first. */
151 struct sk_buff *next;
152 struct sk_buff *prev;
153
154 __u32 qlen;
155 spinlock_t lock;
156};
157
158struct sk_buff;
159
9d4dde52
IC
160/* To allow 64K frame to be packed as single skb without frag_list we
161 * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
162 * buffers which do not start on a page boundary.
163 *
164 * Since GRO uses frags we allocate at least 16 regardless of page
165 * size.
a715dea3 166 */
9d4dde52 167#if (65536/PAGE_SIZE + 1) < 16
eec00954 168#define MAX_SKB_FRAGS 16UL
a715dea3 169#else
9d4dde52 170#define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
a715dea3 171#endif
1da177e4
LT
172
173typedef struct skb_frag_struct skb_frag_t;
174
175struct skb_frag_struct {
a8605c60
IC
176 struct {
177 struct page *p;
178 } page;
cb4dfe56 179#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
a309bb07
DM
180 __u32 page_offset;
181 __u32 size;
cb4dfe56
ED
182#else
183 __u16 page_offset;
184 __u16 size;
185#endif
1da177e4
LT
186};
187
9e903e08
ED
188static inline unsigned int skb_frag_size(const skb_frag_t *frag)
189{
190 return frag->size;
191}
192
193static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
194{
195 frag->size = size;
196}
197
198static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
199{
200 frag->size += delta;
201}
202
203static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
204{
205 frag->size -= delta;
206}
207
ac45f602
PO
208#define HAVE_HW_TIME_STAMP
209
210/**
d3a21be8 211 * struct skb_shared_hwtstamps - hardware time stamps
ac45f602
PO
212 * @hwtstamp: hardware time stamp transformed into duration
213 * since arbitrary point in time
214 * @syststamp: hwtstamp transformed to system time base
215 *
216 * Software time stamps generated by ktime_get_real() are stored in
217 * skb->tstamp. The relation between the different kinds of time
218 * stamps is as follows:
219 *
220 * syststamp and tstamp can be compared against each other in
221 * arbitrary combinations. The accuracy of a
222 * syststamp/tstamp/"syststamp from other device" comparison is
223 * limited by the accuracy of the transformation into system time
224 * base. This depends on the device driver and its underlying
225 * hardware.
226 *
227 * hwtstamps can only be compared against other hwtstamps from
228 * the same device.
229 *
230 * This structure is attached to packets as part of the
231 * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
232 */
233struct skb_shared_hwtstamps {
234 ktime_t hwtstamp;
235 ktime_t syststamp;
236};
237
2244d07b
OH
238/* Definitions for tx_flags in struct skb_shared_info */
239enum {
240 /* generate hardware time stamp */
241 SKBTX_HW_TSTAMP = 1 << 0,
242
243 /* generate software time stamp */
244 SKBTX_SW_TSTAMP = 1 << 1,
245
246 /* device driver is going to provide hardware time stamp */
247 SKBTX_IN_PROGRESS = 1 << 2,
248
a6686f2f 249 /* device driver supports TX zero-copy buffers */
62b1a8ab 250 SKBTX_DEV_ZEROCOPY = 1 << 3,
6e3e939f
JB
251
252 /* generate wifi status information (where possible) */
62b1a8ab 253 SKBTX_WIFI_STATUS = 1 << 4,
c9af6db4
PS
254
255 /* This indicates at least one fragment might be overwritten
256 * (as in vmsplice(), sendfile() ...)
257 * If we need to compute a TX checksum, we'll need to copy
258 * all frags to avoid possible bad checksum
259 */
260 SKBTX_SHARED_FRAG = 1 << 5,
a6686f2f
SM
261};
262
263/*
264 * The callback notifies userspace to release buffers when skb DMA is done in
265 * lower device, the skb last reference should be 0 when calling this.
e19d6763
MT
266 * The zerocopy_success argument is true if zero copy transmit occurred,
267 * false on data copy or out of memory error caused by data copy attempt.
ca8f4fb2
MT
268 * The ctx field is used to track device context.
269 * The desc field is used to track userspace buffer index.
a6686f2f
SM
270 */
271struct ubuf_info {
e19d6763 272 void (*callback)(struct ubuf_info *, bool zerocopy_success);
ca8f4fb2 273 void *ctx;
a6686f2f 274 unsigned long desc;
ac45f602
PO
275};
276
1da177e4
LT
277/* This data is invariant across clones and lives at
278 * the end of the header data, ie. at skb->end.
279 */
280struct skb_shared_info {
9f42f126
IC
281 unsigned char nr_frags;
282 __u8 tx_flags;
7967168c
HX
283 unsigned short gso_size;
284 /* Warning: this field is not always filled in (UFO)! */
285 unsigned short gso_segs;
286 unsigned short gso_type;
1da177e4 287 struct sk_buff *frag_list;
ac45f602 288 struct skb_shared_hwtstamps hwtstamps;
9f42f126 289 __be32 ip6_frag_id;
ec7d2f2c
ED
290
291 /*
292 * Warning : all fields before dataref are cleared in __alloc_skb()
293 */
294 atomic_t dataref;
295
69e3c75f
JB
296 /* Intermediate layers must ensure that destructor_arg
297 * remains valid until skb destructor */
298 void * destructor_arg;
a6686f2f 299
fed66381
ED
300 /* must be last field, see pskb_expand_head() */
301 skb_frag_t frags[MAX_SKB_FRAGS];
1da177e4
LT
302};
303
304/* We divide dataref into two halves. The higher 16 bits hold references
305 * to the payload part of skb->data. The lower 16 bits hold references to
334a8132
PM
306 * the entire skb->data. A clone of a headerless skb holds the length of
307 * the header in skb->hdr_len.
1da177e4
LT
308 *
309 * All users must obey the rule that the skb->data reference count must be
310 * greater than or equal to the payload reference count.
311 *
312 * Holding a reference to the payload part means that the user does not
313 * care about modifications to the header part of skb->data.
314 */
315#define SKB_DATAREF_SHIFT 16
316#define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
317
d179cd12
DM
318
319enum {
320 SKB_FCLONE_UNAVAILABLE,
321 SKB_FCLONE_ORIG,
322 SKB_FCLONE_CLONE,
323};
324
7967168c
HX
325enum {
326 SKB_GSO_TCPV4 = 1 << 0,
f83ef8c0 327 SKB_GSO_UDP = 1 << 1,
576a30eb
HX
328
329 /* This indicates the skb is from an untrusted source. */
330 SKB_GSO_DODGY = 1 << 2,
b0da8537
MC
331
332 /* This indicates the tcp segment has CWR set. */
f83ef8c0
HX
333 SKB_GSO_TCP_ECN = 1 << 3,
334
335 SKB_GSO_TCPV6 = 1 << 4,
01d5b2fc
CL
336
337 SKB_GSO_FCOE = 1 << 5,
68c33163
PS
338
339 SKB_GSO_GRE = 1 << 6,
73136267 340
cb32f511 341 SKB_GSO_IPIP = 1 << 7,
0d89d203 342
61c1db7f 343 SKB_GSO_SIT = 1 << 8,
cb32f511 344
61c1db7f
ED
345 SKB_GSO_UDP_TUNNEL = 1 << 9,
346
347 SKB_GSO_MPLS = 1 << 10,
0f4f4ffa
TH
348
349 SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
4749c09c
TH
350
351 SKB_GSO_GRE_CSUM = 1 << 12,
7967168c
HX
352};
353
2e07fa9c
ACM
354#if BITS_PER_LONG > 32
355#define NET_SKBUFF_DATA_USES_OFFSET 1
356#endif
357
358#ifdef NET_SKBUFF_DATA_USES_OFFSET
359typedef unsigned int sk_buff_data_t;
360#else
361typedef unsigned char *sk_buff_data_t;
362#endif
363
363ec392
ED
364/**
365 * struct skb_mstamp - multi resolution time stamps
366 * @stamp_us: timestamp in us resolution
367 * @stamp_jiffies: timestamp in jiffies
368 */
369struct skb_mstamp {
370 union {
371 u64 v64;
372 struct {
373 u32 stamp_us;
374 u32 stamp_jiffies;
375 };
376 };
377};
378
379/**
380 * skb_mstamp_get - get current timestamp
381 * @cl: place to store timestamps
382 */
383static inline void skb_mstamp_get(struct skb_mstamp *cl)
384{
385 u64 val = local_clock();
386
387 do_div(val, NSEC_PER_USEC);
388 cl->stamp_us = (u32)val;
389 cl->stamp_jiffies = (u32)jiffies;
390}
391
392/**
393 * skb_mstamp_delta - compute the difference in usec between two skb_mstamp
394 * @t1: pointer to newest sample
395 * @t0: pointer to oldest sample
396 */
397static inline u32 skb_mstamp_us_delta(const struct skb_mstamp *t1,
398 const struct skb_mstamp *t0)
399{
400 s32 delta_us = t1->stamp_us - t0->stamp_us;
401 u32 delta_jiffies = t1->stamp_jiffies - t0->stamp_jiffies;
402
403 /* If delta_us is negative, this might be because interval is too big,
404 * or local_clock() drift is too big : fallback using jiffies.
405 */
406 if (delta_us <= 0 ||
407 delta_jiffies >= (INT_MAX / (USEC_PER_SEC / HZ)))
408
409 delta_us = jiffies_to_usecs(delta_jiffies);
410
411 return delta_us;
412}
413
414
1da177e4
LT
415/**
416 * struct sk_buff - socket buffer
417 * @next: Next buffer in list
418 * @prev: Previous buffer in list
363ec392 419 * @tstamp: Time we arrived/left
d84e0bd7 420 * @sk: Socket we are owned by
1da177e4 421 * @dev: Device we arrived on/are leaving by
d84e0bd7 422 * @cb: Control buffer. Free for use by every layer. Put private vars here
7fee226a 423 * @_skb_refdst: destination entry (with norefcount bit)
67be2dd1 424 * @sp: the security path, used for xfrm
1da177e4
LT
425 * @len: Length of actual data
426 * @data_len: Data length
427 * @mac_len: Length of link layer header
334a8132 428 * @hdr_len: writable header length of cloned skb
663ead3b
HX
429 * @csum: Checksum (must include start/offset pair)
430 * @csum_start: Offset from skb->head where checksumming should start
431 * @csum_offset: Offset from csum_start where checksum should be stored
d84e0bd7 432 * @priority: Packet queueing priority
60ff7467 433 * @ignore_df: allow local fragmentation
1da177e4 434 * @cloned: Head may be cloned (check refcnt to be sure)
d84e0bd7 435 * @ip_summed: Driver fed us an IP checksum
1da177e4 436 * @nohdr: Payload reference only, must not modify header
d84e0bd7 437 * @nfctinfo: Relationship of this skb to the connection
1da177e4 438 * @pkt_type: Packet class
c83c2486 439 * @fclone: skbuff clone status
c83c2486 440 * @ipvs_property: skbuff is owned by ipvs
31729363
RD
441 * @peeked: this packet has been seen already, so stats have been
442 * done for it, don't do them again
ba9dda3a 443 * @nf_trace: netfilter packet trace flag
d84e0bd7
DB
444 * @protocol: Packet protocol from driver
445 * @destructor: Destruct function
446 * @nfct: Associated connection, if any
1da177e4 447 * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
8964be4a 448 * @skb_iif: ifindex of device we arrived on
1da177e4
LT
449 * @tc_index: Traffic control index
450 * @tc_verd: traffic control verdict
61b905da 451 * @hash: the packet hash
d84e0bd7 452 * @queue_mapping: Queue mapping for multiqueue devices
553a5672 453 * @ndisc_nodetype: router type (from link layer)
d84e0bd7 454 * @ooo_okay: allow the mapping of a socket to a queue to be changed
61b905da 455 * @l4_hash: indicate hash is a canonical 4-tuple hash over transport
4ca2462e 456 * ports.
6e3e939f
JB
457 * @wifi_acked_valid: wifi_acked was set
458 * @wifi_acked: whether frame was acked on wifi or not
3bdc0eba 459 * @no_fcs: Request NIC to treat last 4 bytes as Ethernet FCS
f4b8ea78
RD
460 * @dma_cookie: a cookie to one of several possible DMA operations
461 * done by skb DMA functions
06021292 462 * @napi_id: id of the NAPI struct this skb came from
984bc16c 463 * @secmark: security marking
d84e0bd7
DB
464 * @mark: Generic packet mark
465 * @dropcount: total number of sk_receive_queue overflows
86a9bad3 466 * @vlan_proto: vlan encapsulation protocol
6aa895b0 467 * @vlan_tci: vlan tag control information
0d89d203 468 * @inner_protocol: Protocol (encapsulation)
6a674e9c
JG
469 * @inner_transport_header: Inner transport layer header (encapsulation)
470 * @inner_network_header: Network layer header (encapsulation)
aefbd2b3 471 * @inner_mac_header: Link layer header (encapsulation)
d84e0bd7
DB
472 * @transport_header: Transport layer header
473 * @network_header: Network layer header
474 * @mac_header: Link layer header
475 * @tail: Tail pointer
476 * @end: End pointer
477 * @head: Head of buffer
478 * @data: Data head pointer
479 * @truesize: Buffer size
480 * @users: User count - see {datagram,tcp}.c
1da177e4
LT
481 */
482
483struct sk_buff {
484 /* These two members must be first. */
485 struct sk_buff *next;
486 struct sk_buff *prev;
487
363ec392
ED
488 union {
489 ktime_t tstamp;
490 struct skb_mstamp skb_mstamp;
491 };
da3f5cf1
FF
492
493 struct sock *sk;
1da177e4 494 struct net_device *dev;
1da177e4 495
1da177e4
LT
496 /*
497 * This is the control buffer. It is free to use for every
498 * layer. Please put your private variables there. If you
499 * want to keep them across layers you have to do a skb_clone()
500 * first. This is owned by whoever has the skb queued ATM.
501 */
da3f5cf1 502 char cb[48] __aligned(8);
1da177e4 503
7fee226a 504 unsigned long _skb_refdst;
da3f5cf1
FF
505#ifdef CONFIG_XFRM
506 struct sec_path *sp;
507#endif
1da177e4 508 unsigned int len,
334a8132
PM
509 data_len;
510 __u16 mac_len,
511 hdr_len;
ff1dcadb
AV
512 union {
513 __wsum csum;
663ead3b
HX
514 struct {
515 __u16 csum_start;
516 __u16 csum_offset;
517 };
ff1dcadb 518 };
1da177e4 519 __u32 priority;
fe55f6d5 520 kmemcheck_bitfield_begin(flags1);
60ff7467 521 __u8 ignore_df:1,
1cbb3380
TG
522 cloned:1,
523 ip_summed:2,
6869c4d8
HW
524 nohdr:1,
525 nfctinfo:3;
d179cd12 526 __u8 pkt_type:3,
b84f4cc9 527 fclone:2,
ba9dda3a 528 ipvs_property:1,
a59322be 529 peeked:1,
ba9dda3a 530 nf_trace:1;
fe55f6d5 531 kmemcheck_bitfield_end(flags1);
4ab408de 532 __be16 protocol;
1da177e4
LT
533
534 void (*destructor)(struct sk_buff *skb);
9fb9cbb1 535#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
5f79e0f9 536 struct nf_conntrack *nfct;
2fc72c7b 537#endif
1da177e4
LT
538#ifdef CONFIG_BRIDGE_NETFILTER
539 struct nf_bridge_info *nf_bridge;
540#endif
f25f4e44 541
8964be4a 542 int skb_iif;
4031ae6e 543
61b905da 544 __u32 hash;
4031ae6e 545
86a9bad3 546 __be16 vlan_proto;
4031ae6e
AD
547 __u16 vlan_tci;
548
1da177e4 549#ifdef CONFIG_NET_SCHED
b6b99eb5 550 __u16 tc_index; /* traffic control index */
1da177e4 551#ifdef CONFIG_NET_CLS_ACT
b6b99eb5 552 __u16 tc_verd; /* traffic control verdict */
1da177e4 553#endif
1da177e4 554#endif
fe55f6d5 555
0a14842f 556 __u16 queue_mapping;
fe55f6d5 557 kmemcheck_bitfield_begin(flags2);
de357cc0 558#ifdef CONFIG_IPV6_NDISC_NODETYPE
8a4eb573 559 __u8 ndisc_nodetype:2;
d0f09804 560#endif
c93bdd0e 561 __u8 pfmemalloc:1;
3853b584 562 __u8 ooo_okay:1;
61b905da 563 __u8 l4_hash:1;
6e3e939f
JB
564 __u8 wifi_acked_valid:1;
565 __u8 wifi_acked:1;
3bdc0eba 566 __u8 no_fcs:1;
d3836f21 567 __u8 head_frag:1;
6a674e9c
JG
568 /* Encapsulation protocol and NIC drivers should use
569 * this flag to indicate to each other if the skb contains
570 * encapsulated packet or not and maybe use the inner packet
571 * headers if needed
572 */
573 __u8 encapsulation:1;
7e2b10c1 574 __u8 encap_hdr_csum:1;
5d0c2b95
TH
575 __u8 csum_valid:1;
576 /* 4/6 bit hole (depending on ndisc_nodetype presence) */
fe55f6d5
VN
577 kmemcheck_bitfield_end(flags2);
578
e0d1095a 579#if defined CONFIG_NET_DMA || defined CONFIG_NET_RX_BUSY_POLL
06021292
ET
580 union {
581 unsigned int napi_id;
582 dma_cookie_t dma_cookie;
583 };
97fc2f08 584#endif
984bc16c
JM
585#ifdef CONFIG_NETWORK_SECMARK
586 __u32 secmark;
587#endif
3b885787
NH
588 union {
589 __u32 mark;
590 __u32 dropcount;
16fad69c 591 __u32 reserved_tailroom;
3b885787 592 };
1da177e4 593
0d89d203 594 __be16 inner_protocol;
1a37e412
SH
595 __u16 inner_transport_header;
596 __u16 inner_network_header;
597 __u16 inner_mac_header;
598 __u16 transport_header;
599 __u16 network_header;
600 __u16 mac_header;
1da177e4 601 /* These elements must be at the end, see alloc_skb() for details. */
27a884dc 602 sk_buff_data_t tail;
4305b541 603 sk_buff_data_t end;
1da177e4 604 unsigned char *head,
4305b541 605 *data;
27a884dc
ACM
606 unsigned int truesize;
607 atomic_t users;
1da177e4
LT
608};
609
610#ifdef __KERNEL__
611/*
612 * Handling routines are only of interest to the kernel
613 */
614#include <linux/slab.h>
615
1da177e4 616
c93bdd0e
MG
617#define SKB_ALLOC_FCLONE 0x01
618#define SKB_ALLOC_RX 0x02
619
620/* Returns true if the skb was allocated from PFMEMALLOC reserves */
621static inline bool skb_pfmemalloc(const struct sk_buff *skb)
622{
623 return unlikely(skb->pfmemalloc);
624}
625
7fee226a
ED
626/*
627 * skb might have a dst pointer attached, refcounted or not.
628 * _skb_refdst low order bit is set if refcount was _not_ taken
629 */
630#define SKB_DST_NOREF 1UL
631#define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
632
633/**
634 * skb_dst - returns skb dst_entry
635 * @skb: buffer
636 *
637 * Returns skb dst_entry, regardless of reference taken or not.
638 */
adf30907
ED
639static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
640{
7fee226a
ED
641 /* If refdst was not refcounted, check we still are in a
642 * rcu_read_lock section
643 */
644 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
645 !rcu_read_lock_held() &&
646 !rcu_read_lock_bh_held());
647 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
adf30907
ED
648}
649
7fee226a
ED
650/**
651 * skb_dst_set - sets skb dst
652 * @skb: buffer
653 * @dst: dst entry
654 *
655 * Sets skb dst, assuming a reference was taken on dst and should
656 * be released by skb_dst_drop()
657 */
adf30907
ED
658static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
659{
7fee226a
ED
660 skb->_skb_refdst = (unsigned long)dst;
661}
662
7965bd4d
JP
663void __skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst,
664 bool force);
932bc4d7
JA
665
666/**
667 * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
668 * @skb: buffer
669 * @dst: dst entry
670 *
671 * Sets skb dst, assuming a reference was not taken on dst.
672 * If dst entry is cached, we do not take reference and dst_release
673 * will be avoided by refdst_drop. If dst entry is not cached, we take
674 * reference, so that last dst_release can destroy the dst immediately.
675 */
676static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
677{
678 __skb_dst_set_noref(skb, dst, false);
679}
680
681/**
682 * skb_dst_set_noref_force - sets skb dst, without taking reference
683 * @skb: buffer
684 * @dst: dst entry
685 *
686 * Sets skb dst, assuming a reference was not taken on dst.
687 * No reference is taken and no dst_release will be called. While for
688 * cached dsts deferred reclaim is a basic feature, for entries that are
689 * not cached it is caller's job to guarantee that last dst_release for
690 * provided dst happens when nobody uses it, eg. after a RCU grace period.
691 */
692static inline void skb_dst_set_noref_force(struct sk_buff *skb,
693 struct dst_entry *dst)
694{
695 __skb_dst_set_noref(skb, dst, true);
696}
7fee226a
ED
697
698/**
25985edc 699 * skb_dst_is_noref - Test if skb dst isn't refcounted
7fee226a
ED
700 * @skb: buffer
701 */
702static inline bool skb_dst_is_noref(const struct sk_buff *skb)
703{
704 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
adf30907
ED
705}
706
511c3f92
ED
707static inline struct rtable *skb_rtable(const struct sk_buff *skb)
708{
adf30907 709 return (struct rtable *)skb_dst(skb);
511c3f92
ED
710}
711
7965bd4d
JP
712void kfree_skb(struct sk_buff *skb);
713void kfree_skb_list(struct sk_buff *segs);
714void skb_tx_error(struct sk_buff *skb);
715void consume_skb(struct sk_buff *skb);
716void __kfree_skb(struct sk_buff *skb);
d7e8883c 717extern struct kmem_cache *skbuff_head_cache;
bad43ca8 718
7965bd4d
JP
719void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
720bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
721 bool *fragstolen, int *delta_truesize);
bad43ca8 722
7965bd4d
JP
723struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
724 int node);
725struct sk_buff *build_skb(void *data, unsigned int frag_size);
d179cd12 726static inline struct sk_buff *alloc_skb(unsigned int size,
dd0fc66f 727 gfp_t priority)
d179cd12 728{
564824b0 729 return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
d179cd12
DM
730}
731
732static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
dd0fc66f 733 gfp_t priority)
d179cd12 734{
c93bdd0e 735 return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
d179cd12
DM
736}
737
7965bd4d 738struct sk_buff *__alloc_skb_head(gfp_t priority, int node);
0ebd0ac5
PM
739static inline struct sk_buff *alloc_skb_head(gfp_t priority)
740{
741 return __alloc_skb_head(priority, -1);
742}
743
7965bd4d
JP
744struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
745int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
746struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
747struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
bad93e9d
OP
748struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
749 gfp_t gfp_mask, bool fclone);
750static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
751 gfp_t gfp_mask)
752{
753 return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
754}
7965bd4d
JP
755
756int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
757struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
758 unsigned int headroom);
759struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
760 int newtailroom, gfp_t priority);
25a91d8d
FD
761int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
762 int offset, int len);
7965bd4d
JP
763int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset,
764 int len);
765int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
766int skb_pad(struct sk_buff *skb, int pad);
ead2ceb0 767#define dev_kfree_skb(a) consume_skb(a)
1da177e4 768
7965bd4d
JP
769int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
770 int getfrag(void *from, char *to, int offset,
771 int len, int odd, struct sk_buff *skb),
772 void *from, int length);
e89e9cf5 773
d94d9fee 774struct skb_seq_state {
677e90ed
TG
775 __u32 lower_offset;
776 __u32 upper_offset;
777 __u32 frag_idx;
778 __u32 stepped_offset;
779 struct sk_buff *root_skb;
780 struct sk_buff *cur_skb;
781 __u8 *frag_data;
782};
783
7965bd4d
JP
784void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
785 unsigned int to, struct skb_seq_state *st);
786unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
787 struct skb_seq_state *st);
788void skb_abort_seq_read(struct skb_seq_state *st);
677e90ed 789
7965bd4d
JP
790unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
791 unsigned int to, struct ts_config *config,
792 struct ts_state *state);
3fc7e8a6 793
09323cc4
TH
794/*
795 * Packet hash types specify the type of hash in skb_set_hash.
796 *
797 * Hash types refer to the protocol layer addresses which are used to
798 * construct a packet's hash. The hashes are used to differentiate or identify
799 * flows of the protocol layer for the hash type. Hash types are either
800 * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
801 *
802 * Properties of hashes:
803 *
804 * 1) Two packets in different flows have different hash values
805 * 2) Two packets in the same flow should have the same hash value
806 *
807 * A hash at a higher layer is considered to be more specific. A driver should
808 * set the most specific hash possible.
809 *
810 * A driver cannot indicate a more specific hash than the layer at which a hash
811 * was computed. For instance an L3 hash cannot be set as an L4 hash.
812 *
813 * A driver may indicate a hash level which is less specific than the
814 * actual layer the hash was computed on. For instance, a hash computed
815 * at L4 may be considered an L3 hash. This should only be done if the
816 * driver can't unambiguously determine that the HW computed the hash at
817 * the higher layer. Note that the "should" in the second property above
818 * permits this.
819 */
820enum pkt_hash_types {
821 PKT_HASH_TYPE_NONE, /* Undefined type */
822 PKT_HASH_TYPE_L2, /* Input: src_MAC, dest_MAC */
823 PKT_HASH_TYPE_L3, /* Input: src_IP, dst_IP */
824 PKT_HASH_TYPE_L4, /* Input: src_IP, dst_IP, src_port, dst_port */
825};
826
827static inline void
828skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
829{
61b905da
TH
830 skb->l4_hash = (type == PKT_HASH_TYPE_L4);
831 skb->hash = hash;
09323cc4
TH
832}
833
3958afa1
TH
834void __skb_get_hash(struct sk_buff *skb);
835static inline __u32 skb_get_hash(struct sk_buff *skb)
bfb564e7 836{
61b905da 837 if (!skb->l4_hash)
3958afa1 838 __skb_get_hash(skb);
bfb564e7 839
61b905da 840 return skb->hash;
bfb564e7
KK
841}
842
57bdf7f4
TH
843static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
844{
61b905da 845 return skb->hash;
57bdf7f4
TH
846}
847
7539fadc
TH
848static inline void skb_clear_hash(struct sk_buff *skb)
849{
61b905da
TH
850 skb->hash = 0;
851 skb->l4_hash = 0;
7539fadc
TH
852}
853
854static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
855{
61b905da 856 if (!skb->l4_hash)
7539fadc
TH
857 skb_clear_hash(skb);
858}
859
3df7a74e
TH
860static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
861{
61b905da
TH
862 to->hash = from->hash;
863 to->l4_hash = from->l4_hash;
3df7a74e
TH
864};
865
4305b541
ACM
866#ifdef NET_SKBUFF_DATA_USES_OFFSET
867static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
868{
869 return skb->head + skb->end;
870}
ec47ea82
AD
871
872static inline unsigned int skb_end_offset(const struct sk_buff *skb)
873{
874 return skb->end;
875}
4305b541
ACM
876#else
877static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
878{
879 return skb->end;
880}
ec47ea82
AD
881
882static inline unsigned int skb_end_offset(const struct sk_buff *skb)
883{
884 return skb->end - skb->head;
885}
4305b541
ACM
886#endif
887
1da177e4 888/* Internal */
4305b541 889#define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
1da177e4 890
ac45f602
PO
891static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
892{
893 return &skb_shinfo(skb)->hwtstamps;
894}
895
1da177e4
LT
896/**
897 * skb_queue_empty - check if a queue is empty
898 * @list: queue head
899 *
900 * Returns true if the queue is empty, false otherwise.
901 */
902static inline int skb_queue_empty(const struct sk_buff_head *list)
903{
fd44b93c 904 return list->next == (const struct sk_buff *) list;
1da177e4
LT
905}
906
fc7ebb21
DM
907/**
908 * skb_queue_is_last - check if skb is the last entry in the queue
909 * @list: queue head
910 * @skb: buffer
911 *
912 * Returns true if @skb is the last buffer on the list.
913 */
914static inline bool skb_queue_is_last(const struct sk_buff_head *list,
915 const struct sk_buff *skb)
916{
fd44b93c 917 return skb->next == (const struct sk_buff *) list;
fc7ebb21
DM
918}
919
832d11c5
IJ
920/**
921 * skb_queue_is_first - check if skb is the first entry in the queue
922 * @list: queue head
923 * @skb: buffer
924 *
925 * Returns true if @skb is the first buffer on the list.
926 */
927static inline bool skb_queue_is_first(const struct sk_buff_head *list,
928 const struct sk_buff *skb)
929{
fd44b93c 930 return skb->prev == (const struct sk_buff *) list;
832d11c5
IJ
931}
932
249c8b42
DM
933/**
934 * skb_queue_next - return the next packet in the queue
935 * @list: queue head
936 * @skb: current buffer
937 *
938 * Return the next packet in @list after @skb. It is only valid to
939 * call this if skb_queue_is_last() evaluates to false.
940 */
941static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
942 const struct sk_buff *skb)
943{
944 /* This BUG_ON may seem severe, but if we just return then we
945 * are going to dereference garbage.
946 */
947 BUG_ON(skb_queue_is_last(list, skb));
948 return skb->next;
949}
950
832d11c5
IJ
951/**
952 * skb_queue_prev - return the prev packet in the queue
953 * @list: queue head
954 * @skb: current buffer
955 *
956 * Return the prev packet in @list before @skb. It is only valid to
957 * call this if skb_queue_is_first() evaluates to false.
958 */
959static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
960 const struct sk_buff *skb)
961{
962 /* This BUG_ON may seem severe, but if we just return then we
963 * are going to dereference garbage.
964 */
965 BUG_ON(skb_queue_is_first(list, skb));
966 return skb->prev;
967}
968
1da177e4
LT
969/**
970 * skb_get - reference buffer
971 * @skb: buffer to reference
972 *
973 * Makes another reference to a socket buffer and returns a pointer
974 * to the buffer.
975 */
976static inline struct sk_buff *skb_get(struct sk_buff *skb)
977{
978 atomic_inc(&skb->users);
979 return skb;
980}
981
982/*
983 * If users == 1, we are the only owner and are can avoid redundant
984 * atomic change.
985 */
986
1da177e4
LT
987/**
988 * skb_cloned - is the buffer a clone
989 * @skb: buffer to check
990 *
991 * Returns true if the buffer was generated with skb_clone() and is
992 * one of multiple shared copies of the buffer. Cloned buffers are
993 * shared data so must not be written to under normal circumstances.
994 */
995static inline int skb_cloned(const struct sk_buff *skb)
996{
997 return skb->cloned &&
998 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
999}
1000
14bbd6a5
PS
1001static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
1002{
1003 might_sleep_if(pri & __GFP_WAIT);
1004
1005 if (skb_cloned(skb))
1006 return pskb_expand_head(skb, 0, 0, pri);
1007
1008 return 0;
1009}
1010
1da177e4
LT
1011/**
1012 * skb_header_cloned - is the header a clone
1013 * @skb: buffer to check
1014 *
1015 * Returns true if modifying the header part of the buffer requires
1016 * the data to be copied.
1017 */
1018static inline int skb_header_cloned(const struct sk_buff *skb)
1019{
1020 int dataref;
1021
1022 if (!skb->cloned)
1023 return 0;
1024
1025 dataref = atomic_read(&skb_shinfo(skb)->dataref);
1026 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
1027 return dataref != 1;
1028}
1029
1030/**
1031 * skb_header_release - release reference to header
1032 * @skb: buffer to operate on
1033 *
1034 * Drop a reference to the header part of the buffer. This is done
1035 * by acquiring a payload reference. You must not read from the header
1036 * part of skb->data after this.
1037 */
1038static inline void skb_header_release(struct sk_buff *skb)
1039{
1040 BUG_ON(skb->nohdr);
1041 skb->nohdr = 1;
1042 atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref);
1043}
1044
1045/**
1046 * skb_shared - is the buffer shared
1047 * @skb: buffer to check
1048 *
1049 * Returns true if more than one person has a reference to this
1050 * buffer.
1051 */
1052static inline int skb_shared(const struct sk_buff *skb)
1053{
1054 return atomic_read(&skb->users) != 1;
1055}
1056
1057/**
1058 * skb_share_check - check if buffer is shared and if so clone it
1059 * @skb: buffer to check
1060 * @pri: priority for memory allocation
1061 *
1062 * If the buffer is shared the buffer is cloned and the old copy
1063 * drops a reference. A new clone with a single reference is returned.
1064 * If the buffer is not shared the original buffer is returned. When
1065 * being called from interrupt status or with spinlocks held pri must
1066 * be GFP_ATOMIC.
1067 *
1068 * NULL is returned on a memory allocation failure.
1069 */
47061bc4 1070static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
1da177e4
LT
1071{
1072 might_sleep_if(pri & __GFP_WAIT);
1073 if (skb_shared(skb)) {
1074 struct sk_buff *nskb = skb_clone(skb, pri);
47061bc4
ED
1075
1076 if (likely(nskb))
1077 consume_skb(skb);
1078 else
1079 kfree_skb(skb);
1da177e4
LT
1080 skb = nskb;
1081 }
1082 return skb;
1083}
1084
1085/*
1086 * Copy shared buffers into a new sk_buff. We effectively do COW on
1087 * packets to handle cases where we have a local reader and forward
1088 * and a couple of other messy ones. The normal one is tcpdumping
1089 * a packet thats being forwarded.
1090 */
1091
1092/**
1093 * skb_unshare - make a copy of a shared buffer
1094 * @skb: buffer to check
1095 * @pri: priority for memory allocation
1096 *
1097 * If the socket buffer is a clone then this function creates a new
1098 * copy of the data, drops a reference count on the old copy and returns
1099 * the new copy with the reference count at 1. If the buffer is not a clone
1100 * the original buffer is returned. When called with a spinlock held or
1101 * from interrupt state @pri must be %GFP_ATOMIC
1102 *
1103 * %NULL is returned on a memory allocation failure.
1104 */
e2bf521d 1105static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
dd0fc66f 1106 gfp_t pri)
1da177e4
LT
1107{
1108 might_sleep_if(pri & __GFP_WAIT);
1109 if (skb_cloned(skb)) {
1110 struct sk_buff *nskb = skb_copy(skb, pri);
1111 kfree_skb(skb); /* Free our shared copy */
1112 skb = nskb;
1113 }
1114 return skb;
1115}
1116
1117/**
1a5778aa 1118 * skb_peek - peek at the head of an &sk_buff_head
1da177e4
LT
1119 * @list_: list to peek at
1120 *
1121 * Peek an &sk_buff. Unlike most other operations you _MUST_
1122 * be careful with this one. A peek leaves the buffer on the
1123 * list and someone else may run off with it. You must hold
1124 * the appropriate locks or have a private queue to do this.
1125 *
1126 * Returns %NULL for an empty list or a pointer to the head element.
1127 * The reference count is not incremented and the reference is therefore
1128 * volatile. Use with caution.
1129 */
05bdd2f1 1130static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
1da177e4 1131{
18d07000
ED
1132 struct sk_buff *skb = list_->next;
1133
1134 if (skb == (struct sk_buff *)list_)
1135 skb = NULL;
1136 return skb;
1da177e4
LT
1137}
1138
da5ef6e5
PE
1139/**
1140 * skb_peek_next - peek skb following the given one from a queue
1141 * @skb: skb to start from
1142 * @list_: list to peek at
1143 *
1144 * Returns %NULL when the end of the list is met or a pointer to the
1145 * next element. The reference count is not incremented and the
1146 * reference is therefore volatile. Use with caution.
1147 */
1148static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
1149 const struct sk_buff_head *list_)
1150{
1151 struct sk_buff *next = skb->next;
18d07000 1152
da5ef6e5
PE
1153 if (next == (struct sk_buff *)list_)
1154 next = NULL;
1155 return next;
1156}
1157
1da177e4 1158/**
1a5778aa 1159 * skb_peek_tail - peek at the tail of an &sk_buff_head
1da177e4
LT
1160 * @list_: list to peek at
1161 *
1162 * Peek an &sk_buff. Unlike most other operations you _MUST_
1163 * be careful with this one. A peek leaves the buffer on the
1164 * list and someone else may run off with it. You must hold
1165 * the appropriate locks or have a private queue to do this.
1166 *
1167 * Returns %NULL for an empty list or a pointer to the tail element.
1168 * The reference count is not incremented and the reference is therefore
1169 * volatile. Use with caution.
1170 */
05bdd2f1 1171static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
1da177e4 1172{
18d07000
ED
1173 struct sk_buff *skb = list_->prev;
1174
1175 if (skb == (struct sk_buff *)list_)
1176 skb = NULL;
1177 return skb;
1178
1da177e4
LT
1179}
1180
1181/**
1182 * skb_queue_len - get queue length
1183 * @list_: list to measure
1184 *
1185 * Return the length of an &sk_buff queue.
1186 */
1187static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
1188{
1189 return list_->qlen;
1190}
1191
67fed459
DM
1192/**
1193 * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
1194 * @list: queue to initialize
1195 *
1196 * This initializes only the list and queue length aspects of
1197 * an sk_buff_head object. This allows to initialize the list
1198 * aspects of an sk_buff_head without reinitializing things like
1199 * the spinlock. It can also be used for on-stack sk_buff_head
1200 * objects where the spinlock is known to not be used.
1201 */
1202static inline void __skb_queue_head_init(struct sk_buff_head *list)
1203{
1204 list->prev = list->next = (struct sk_buff *)list;
1205 list->qlen = 0;
1206}
1207
76f10ad0
AV
1208/*
1209 * This function creates a split out lock class for each invocation;
1210 * this is needed for now since a whole lot of users of the skb-queue
1211 * infrastructure in drivers have different locking usage (in hardirq)
1212 * than the networking core (in softirq only). In the long run either the
1213 * network layer or drivers should need annotation to consolidate the
1214 * main types of usage into 3 classes.
1215 */
1da177e4
LT
1216static inline void skb_queue_head_init(struct sk_buff_head *list)
1217{
1218 spin_lock_init(&list->lock);
67fed459 1219 __skb_queue_head_init(list);
1da177e4
LT
1220}
1221
c2ecba71
PE
1222static inline void skb_queue_head_init_class(struct sk_buff_head *list,
1223 struct lock_class_key *class)
1224{
1225 skb_queue_head_init(list);
1226 lockdep_set_class(&list->lock, class);
1227}
1228
1da177e4 1229/*
bf299275 1230 * Insert an sk_buff on a list.
1da177e4
LT
1231 *
1232 * The "__skb_xxxx()" functions are the non-atomic ones that
1233 * can only be called with interrupts disabled.
1234 */
7965bd4d
JP
1235void skb_insert(struct sk_buff *old, struct sk_buff *newsk,
1236 struct sk_buff_head *list);
bf299275
GR
1237static inline void __skb_insert(struct sk_buff *newsk,
1238 struct sk_buff *prev, struct sk_buff *next,
1239 struct sk_buff_head *list)
1240{
1241 newsk->next = next;
1242 newsk->prev = prev;
1243 next->prev = prev->next = newsk;
1244 list->qlen++;
1245}
1da177e4 1246
67fed459
DM
1247static inline void __skb_queue_splice(const struct sk_buff_head *list,
1248 struct sk_buff *prev,
1249 struct sk_buff *next)
1250{
1251 struct sk_buff *first = list->next;
1252 struct sk_buff *last = list->prev;
1253
1254 first->prev = prev;
1255 prev->next = first;
1256
1257 last->next = next;
1258 next->prev = last;
1259}
1260
1261/**
1262 * skb_queue_splice - join two skb lists, this is designed for stacks
1263 * @list: the new list to add
1264 * @head: the place to add it in the first list
1265 */
1266static inline void skb_queue_splice(const struct sk_buff_head *list,
1267 struct sk_buff_head *head)
1268{
1269 if (!skb_queue_empty(list)) {
1270 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1d4a31dd 1271 head->qlen += list->qlen;
67fed459
DM
1272 }
1273}
1274
1275/**
d9619496 1276 * skb_queue_splice_init - join two skb lists and reinitialise the emptied list
67fed459
DM
1277 * @list: the new list to add
1278 * @head: the place to add it in the first list
1279 *
1280 * The list at @list is reinitialised
1281 */
1282static inline void skb_queue_splice_init(struct sk_buff_head *list,
1283 struct sk_buff_head *head)
1284{
1285 if (!skb_queue_empty(list)) {
1286 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1d4a31dd 1287 head->qlen += list->qlen;
67fed459
DM
1288 __skb_queue_head_init(list);
1289 }
1290}
1291
1292/**
1293 * skb_queue_splice_tail - join two skb lists, each list being a queue
1294 * @list: the new list to add
1295 * @head: the place to add it in the first list
1296 */
1297static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
1298 struct sk_buff_head *head)
1299{
1300 if (!skb_queue_empty(list)) {
1301 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1d4a31dd 1302 head->qlen += list->qlen;
67fed459
DM
1303 }
1304}
1305
1306/**
d9619496 1307 * skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
67fed459
DM
1308 * @list: the new list to add
1309 * @head: the place to add it in the first list
1310 *
1311 * Each of the lists is a queue.
1312 * The list at @list is reinitialised
1313 */
1314static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
1315 struct sk_buff_head *head)
1316{
1317 if (!skb_queue_empty(list)) {
1318 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1d4a31dd 1319 head->qlen += list->qlen;
67fed459
DM
1320 __skb_queue_head_init(list);
1321 }
1322}
1323
1da177e4 1324/**
300ce174 1325 * __skb_queue_after - queue a buffer at the list head
1da177e4 1326 * @list: list to use
300ce174 1327 * @prev: place after this buffer
1da177e4
LT
1328 * @newsk: buffer to queue
1329 *
300ce174 1330 * Queue a buffer int the middle of a list. This function takes no locks
1da177e4
LT
1331 * and you must therefore hold required locks before calling it.
1332 *
1333 * A buffer cannot be placed on two lists at the same time.
1334 */
300ce174
SH
1335static inline void __skb_queue_after(struct sk_buff_head *list,
1336 struct sk_buff *prev,
1337 struct sk_buff *newsk)
1da177e4 1338{
bf299275 1339 __skb_insert(newsk, prev, prev->next, list);
1da177e4
LT
1340}
1341
7965bd4d
JP
1342void skb_append(struct sk_buff *old, struct sk_buff *newsk,
1343 struct sk_buff_head *list);
7de6c033 1344
f5572855
GR
1345static inline void __skb_queue_before(struct sk_buff_head *list,
1346 struct sk_buff *next,
1347 struct sk_buff *newsk)
1348{
1349 __skb_insert(newsk, next->prev, next, list);
1350}
1351
300ce174
SH
1352/**
1353 * __skb_queue_head - queue a buffer at the list head
1354 * @list: list to use
1355 * @newsk: buffer to queue
1356 *
1357 * Queue a buffer at the start of a list. This function takes no locks
1358 * and you must therefore hold required locks before calling it.
1359 *
1360 * A buffer cannot be placed on two lists at the same time.
1361 */
7965bd4d 1362void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
300ce174
SH
1363static inline void __skb_queue_head(struct sk_buff_head *list,
1364 struct sk_buff *newsk)
1365{
1366 __skb_queue_after(list, (struct sk_buff *)list, newsk);
1367}
1368
1da177e4
LT
1369/**
1370 * __skb_queue_tail - queue a buffer at the list tail
1371 * @list: list to use
1372 * @newsk: buffer to queue
1373 *
1374 * Queue a buffer at the end of a list. This function takes no locks
1375 * and you must therefore hold required locks before calling it.
1376 *
1377 * A buffer cannot be placed on two lists at the same time.
1378 */
7965bd4d 1379void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
1da177e4
LT
1380static inline void __skb_queue_tail(struct sk_buff_head *list,
1381 struct sk_buff *newsk)
1382{
f5572855 1383 __skb_queue_before(list, (struct sk_buff *)list, newsk);
1da177e4
LT
1384}
1385
1da177e4
LT
1386/*
1387 * remove sk_buff from list. _Must_ be called atomically, and with
1388 * the list known..
1389 */
7965bd4d 1390void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
1da177e4
LT
1391static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1392{
1393 struct sk_buff *next, *prev;
1394
1395 list->qlen--;
1396 next = skb->next;
1397 prev = skb->prev;
1398 skb->next = skb->prev = NULL;
1da177e4
LT
1399 next->prev = prev;
1400 prev->next = next;
1401}
1402
f525c06d
GR
1403/**
1404 * __skb_dequeue - remove from the head of the queue
1405 * @list: list to dequeue from
1406 *
1407 * Remove the head of the list. This function does not take any locks
1408 * so must be used with appropriate locks held only. The head item is
1409 * returned or %NULL if the list is empty.
1410 */
7965bd4d 1411struct sk_buff *skb_dequeue(struct sk_buff_head *list);
f525c06d
GR
1412static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
1413{
1414 struct sk_buff *skb = skb_peek(list);
1415 if (skb)
1416 __skb_unlink(skb, list);
1417 return skb;
1418}
1da177e4
LT
1419
1420/**
1421 * __skb_dequeue_tail - remove from the tail of the queue
1422 * @list: list to dequeue from
1423 *
1424 * Remove the tail of the list. This function does not take any locks
1425 * so must be used with appropriate locks held only. The tail item is
1426 * returned or %NULL if the list is empty.
1427 */
7965bd4d 1428struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
1da177e4
LT
1429static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
1430{
1431 struct sk_buff *skb = skb_peek_tail(list);
1432 if (skb)
1433 __skb_unlink(skb, list);
1434 return skb;
1435}
1436
1437
bdcc0924 1438static inline bool skb_is_nonlinear(const struct sk_buff *skb)
1da177e4
LT
1439{
1440 return skb->data_len;
1441}
1442
1443static inline unsigned int skb_headlen(const struct sk_buff *skb)
1444{
1445 return skb->len - skb->data_len;
1446}
1447
1448static inline int skb_pagelen(const struct sk_buff *skb)
1449{
1450 int i, len = 0;
1451
1452 for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--)
9e903e08 1453 len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
1da177e4
LT
1454 return len + skb_headlen(skb);
1455}
1456
131ea667
IC
1457/**
1458 * __skb_fill_page_desc - initialise a paged fragment in an skb
1459 * @skb: buffer containing fragment to be initialised
1460 * @i: paged fragment index to initialise
1461 * @page: the page to use for this fragment
1462 * @off: the offset to the data with @page
1463 * @size: the length of the data
1464 *
1465 * Initialises the @i'th fragment of @skb to point to &size bytes at
1466 * offset @off within @page.
1467 *
1468 * Does not take any additional reference on the fragment.
1469 */
1470static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
1471 struct page *page, int off, int size)
1da177e4
LT
1472{
1473 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1474
c48a11c7
MG
1475 /*
1476 * Propagate page->pfmemalloc to the skb if we can. The problem is
1477 * that not all callers have unique ownership of the page. If
1478 * pfmemalloc is set, we check the mapping as a mapping implies
1479 * page->index is set (index and pfmemalloc share space).
1480 * If it's a valid mapping, we cannot use page->pfmemalloc but we
1481 * do not lose pfmemalloc information as the pages would not be
1482 * allocated using __GFP_MEMALLOC.
1483 */
a8605c60 1484 frag->page.p = page;
1da177e4 1485 frag->page_offset = off;
9e903e08 1486 skb_frag_size_set(frag, size);
cca7af38
PE
1487
1488 page = compound_head(page);
1489 if (page->pfmemalloc && !page->mapping)
1490 skb->pfmemalloc = true;
131ea667
IC
1491}
1492
1493/**
1494 * skb_fill_page_desc - initialise a paged fragment in an skb
1495 * @skb: buffer containing fragment to be initialised
1496 * @i: paged fragment index to initialise
1497 * @page: the page to use for this fragment
1498 * @off: the offset to the data with @page
1499 * @size: the length of the data
1500 *
1501 * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
bc32383c 1502 * @skb to point to @size bytes at offset @off within @page. In
131ea667
IC
1503 * addition updates @skb such that @i is the last fragment.
1504 *
1505 * Does not take any additional reference on the fragment.
1506 */
1507static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
1508 struct page *page, int off, int size)
1509{
1510 __skb_fill_page_desc(skb, i, page, off, size);
1da177e4
LT
1511 skb_shinfo(skb)->nr_frags = i + 1;
1512}
1513
7965bd4d
JP
1514void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
1515 int size, unsigned int truesize);
654bed16 1516
f8e617e1
JW
1517void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
1518 unsigned int truesize);
1519
1da177e4 1520#define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags)
21dc3301 1521#define SKB_FRAG_ASSERT(skb) BUG_ON(skb_has_frag_list(skb))
1da177e4
LT
1522#define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb))
1523
27a884dc
ACM
1524#ifdef NET_SKBUFF_DATA_USES_OFFSET
1525static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1526{
1527 return skb->head + skb->tail;
1528}
1529
1530static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1531{
1532 skb->tail = skb->data - skb->head;
1533}
1534
1535static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1536{
1537 skb_reset_tail_pointer(skb);
1538 skb->tail += offset;
1539}
7cc46190 1540
27a884dc
ACM
1541#else /* NET_SKBUFF_DATA_USES_OFFSET */
1542static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1543{
1544 return skb->tail;
1545}
1546
1547static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1548{
1549 skb->tail = skb->data;
1550}
1551
1552static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1553{
1554 skb->tail = skb->data + offset;
1555}
4305b541 1556
27a884dc
ACM
1557#endif /* NET_SKBUFF_DATA_USES_OFFSET */
1558
1da177e4
LT
1559/*
1560 * Add data to an sk_buff
1561 */
0c7ddf36 1562unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
7965bd4d 1563unsigned char *skb_put(struct sk_buff *skb, unsigned int len);
1da177e4
LT
1564static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
1565{
27a884dc 1566 unsigned char *tmp = skb_tail_pointer(skb);
1da177e4
LT
1567 SKB_LINEAR_ASSERT(skb);
1568 skb->tail += len;
1569 skb->len += len;
1570 return tmp;
1571}
1572
7965bd4d 1573unsigned char *skb_push(struct sk_buff *skb, unsigned int len);
1da177e4
LT
1574static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len)
1575{
1576 skb->data -= len;
1577 skb->len += len;
1578 return skb->data;
1579}
1580
7965bd4d 1581unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
1da177e4
LT
1582static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
1583{
1584 skb->len -= len;
1585 BUG_ON(skb->len < skb->data_len);
1586 return skb->data += len;
1587}
1588
47d29646
DM
1589static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len)
1590{
1591 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
1592}
1593
7965bd4d 1594unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
1da177e4
LT
1595
1596static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len)
1597{
1598 if (len > skb_headlen(skb) &&
987c402a 1599 !__pskb_pull_tail(skb, len - skb_headlen(skb)))
1da177e4
LT
1600 return NULL;
1601 skb->len -= len;
1602 return skb->data += len;
1603}
1604
1605static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len)
1606{
1607 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
1608}
1609
1610static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len)
1611{
1612 if (likely(len <= skb_headlen(skb)))
1613 return 1;
1614 if (unlikely(len > skb->len))
1615 return 0;
987c402a 1616 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
1da177e4
LT
1617}
1618
1619/**
1620 * skb_headroom - bytes at buffer head
1621 * @skb: buffer to check
1622 *
1623 * Return the number of bytes of free space at the head of an &sk_buff.
1624 */
c2636b4d 1625static inline unsigned int skb_headroom(const struct sk_buff *skb)
1da177e4
LT
1626{
1627 return skb->data - skb->head;
1628}
1629
1630/**
1631 * skb_tailroom - bytes at buffer end
1632 * @skb: buffer to check
1633 *
1634 * Return the number of bytes of free space at the tail of an sk_buff
1635 */
1636static inline int skb_tailroom(const struct sk_buff *skb)
1637{
4305b541 1638 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
1da177e4
LT
1639}
1640
a21d4572
ED
1641/**
1642 * skb_availroom - bytes at buffer end
1643 * @skb: buffer to check
1644 *
1645 * Return the number of bytes of free space at the tail of an sk_buff
1646 * allocated by sk_stream_alloc()
1647 */
1648static inline int skb_availroom(const struct sk_buff *skb)
1649{
16fad69c
ED
1650 if (skb_is_nonlinear(skb))
1651 return 0;
1652
1653 return skb->end - skb->tail - skb->reserved_tailroom;
a21d4572
ED
1654}
1655
1da177e4
LT
1656/**
1657 * skb_reserve - adjust headroom
1658 * @skb: buffer to alter
1659 * @len: bytes to move
1660 *
1661 * Increase the headroom of an empty &sk_buff by reducing the tail
1662 * room. This is only allowed for an empty buffer.
1663 */
8243126c 1664static inline void skb_reserve(struct sk_buff *skb, int len)
1da177e4
LT
1665{
1666 skb->data += len;
1667 skb->tail += len;
1668}
1669
6a674e9c
JG
1670static inline void skb_reset_inner_headers(struct sk_buff *skb)
1671{
aefbd2b3 1672 skb->inner_mac_header = skb->mac_header;
6a674e9c
JG
1673 skb->inner_network_header = skb->network_header;
1674 skb->inner_transport_header = skb->transport_header;
1675}
1676
0b5c9db1
JP
1677static inline void skb_reset_mac_len(struct sk_buff *skb)
1678{
1679 skb->mac_len = skb->network_header - skb->mac_header;
1680}
1681
6a674e9c
JG
1682static inline unsigned char *skb_inner_transport_header(const struct sk_buff
1683 *skb)
1684{
1685 return skb->head + skb->inner_transport_header;
1686}
1687
1688static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
1689{
1690 skb->inner_transport_header = skb->data - skb->head;
1691}
1692
1693static inline void skb_set_inner_transport_header(struct sk_buff *skb,
1694 const int offset)
1695{
1696 skb_reset_inner_transport_header(skb);
1697 skb->inner_transport_header += offset;
1698}
1699
1700static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
1701{
1702 return skb->head + skb->inner_network_header;
1703}
1704
1705static inline void skb_reset_inner_network_header(struct sk_buff *skb)
1706{
1707 skb->inner_network_header = skb->data - skb->head;
1708}
1709
1710static inline void skb_set_inner_network_header(struct sk_buff *skb,
1711 const int offset)
1712{
1713 skb_reset_inner_network_header(skb);
1714 skb->inner_network_header += offset;
1715}
1716
aefbd2b3
PS
1717static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
1718{
1719 return skb->head + skb->inner_mac_header;
1720}
1721
1722static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
1723{
1724 skb->inner_mac_header = skb->data - skb->head;
1725}
1726
1727static inline void skb_set_inner_mac_header(struct sk_buff *skb,
1728 const int offset)
1729{
1730 skb_reset_inner_mac_header(skb);
1731 skb->inner_mac_header += offset;
1732}
fda55eca
ED
1733static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
1734{
35d04610 1735 return skb->transport_header != (typeof(skb->transport_header))~0U;
fda55eca
ED
1736}
1737
9c70220b
ACM
1738static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1739{
2e07fa9c 1740 return skb->head + skb->transport_header;
9c70220b
ACM
1741}
1742
badff6d0
ACM
1743static inline void skb_reset_transport_header(struct sk_buff *skb)
1744{
2e07fa9c 1745 skb->transport_header = skb->data - skb->head;
badff6d0
ACM
1746}
1747
967b05f6
ACM
1748static inline void skb_set_transport_header(struct sk_buff *skb,
1749 const int offset)
1750{
2e07fa9c
ACM
1751 skb_reset_transport_header(skb);
1752 skb->transport_header += offset;
ea2ae17d
ACM
1753}
1754
d56f90a7
ACM
1755static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1756{
2e07fa9c 1757 return skb->head + skb->network_header;
d56f90a7
ACM
1758}
1759
c1d2bbe1
ACM
1760static inline void skb_reset_network_header(struct sk_buff *skb)
1761{
2e07fa9c 1762 skb->network_header = skb->data - skb->head;
c1d2bbe1
ACM
1763}
1764
c14d2450
ACM
1765static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1766{
2e07fa9c
ACM
1767 skb_reset_network_header(skb);
1768 skb->network_header += offset;
c14d2450
ACM
1769}
1770
2e07fa9c 1771static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
bbe735e4 1772{
2e07fa9c 1773 return skb->head + skb->mac_header;
bbe735e4
ACM
1774}
1775
2e07fa9c 1776static inline int skb_mac_header_was_set(const struct sk_buff *skb)
cfe1fc77 1777{
35d04610 1778 return skb->mac_header != (typeof(skb->mac_header))~0U;
2e07fa9c
ACM
1779}
1780
1781static inline void skb_reset_mac_header(struct sk_buff *skb)
1782{
1783 skb->mac_header = skb->data - skb->head;
1784}
1785
1786static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1787{
1788 skb_reset_mac_header(skb);
1789 skb->mac_header += offset;
1790}
1791
0e3da5bb
TT
1792static inline void skb_pop_mac_header(struct sk_buff *skb)
1793{
1794 skb->mac_header = skb->network_header;
1795}
1796
fbbdb8f0
YX
1797static inline void skb_probe_transport_header(struct sk_buff *skb,
1798 const int offset_hint)
1799{
1800 struct flow_keys keys;
1801
1802 if (skb_transport_header_was_set(skb))
1803 return;
1804 else if (skb_flow_dissect(skb, &keys))
1805 skb_set_transport_header(skb, keys.thoff);
1806 else
1807 skb_set_transport_header(skb, offset_hint);
1808}
1809
03606895
ED
1810static inline void skb_mac_header_rebuild(struct sk_buff *skb)
1811{
1812 if (skb_mac_header_was_set(skb)) {
1813 const unsigned char *old_mac = skb_mac_header(skb);
1814
1815 skb_set_mac_header(skb, -skb->mac_len);
1816 memmove(skb_mac_header(skb), old_mac, skb->mac_len);
1817 }
1818}
1819
04fb451e
MM
1820static inline int skb_checksum_start_offset(const struct sk_buff *skb)
1821{
1822 return skb->csum_start - skb_headroom(skb);
1823}
1824
2e07fa9c
ACM
1825static inline int skb_transport_offset(const struct sk_buff *skb)
1826{
1827 return skb_transport_header(skb) - skb->data;
1828}
1829
1830static inline u32 skb_network_header_len(const struct sk_buff *skb)
1831{
1832 return skb->transport_header - skb->network_header;
1833}
1834
6a674e9c
JG
1835static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
1836{
1837 return skb->inner_transport_header - skb->inner_network_header;
1838}
1839
2e07fa9c
ACM
1840static inline int skb_network_offset(const struct sk_buff *skb)
1841{
1842 return skb_network_header(skb) - skb->data;
1843}
48d49d0c 1844
6a674e9c
JG
1845static inline int skb_inner_network_offset(const struct sk_buff *skb)
1846{
1847 return skb_inner_network_header(skb) - skb->data;
1848}
1849
f9599ce1
CG
1850static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
1851{
1852 return pskb_may_pull(skb, skb_network_offset(skb) + len);
1853}
1854
1da177e4
LT
1855/*
1856 * CPUs often take a performance hit when accessing unaligned memory
1857 * locations. The actual performance hit varies, it can be small if the
1858 * hardware handles it or large if we have to take an exception and fix it
1859 * in software.
1860 *
1861 * Since an ethernet header is 14 bytes network drivers often end up with
1862 * the IP header at an unaligned offset. The IP header can be aligned by
1863 * shifting the start of the packet by 2 bytes. Drivers should do this
1864 * with:
1865 *
8660c124 1866 * skb_reserve(skb, NET_IP_ALIGN);
1da177e4
LT
1867 *
1868 * The downside to this alignment of the IP header is that the DMA is now
1869 * unaligned. On some architectures the cost of an unaligned DMA is high
1870 * and this cost outweighs the gains made by aligning the IP header.
8660c124 1871 *
1da177e4
LT
1872 * Since this trade off varies between architectures, we allow NET_IP_ALIGN
1873 * to be overridden.
1874 */
1875#ifndef NET_IP_ALIGN
1876#define NET_IP_ALIGN 2
1877#endif
1878
025be81e
AB
1879/*
1880 * The networking layer reserves some headroom in skb data (via
1881 * dev_alloc_skb). This is used to avoid having to reallocate skb data when
1882 * the header has to grow. In the default case, if the header has to grow
d6301d3d 1883 * 32 bytes or less we avoid the reallocation.
025be81e
AB
1884 *
1885 * Unfortunately this headroom changes the DMA alignment of the resulting
1886 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
1887 * on some architectures. An architecture can override this value,
1888 * perhaps setting it to a cacheline in size (since that will maintain
1889 * cacheline alignment of the DMA). It must be a power of 2.
1890 *
d6301d3d 1891 * Various parts of the networking layer expect at least 32 bytes of
025be81e 1892 * headroom, you should not reduce this.
5933dd2f
ED
1893 *
1894 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
1895 * to reduce average number of cache lines per packet.
1896 * get_rps_cpus() for example only access one 64 bytes aligned block :
18e8c134 1897 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
025be81e
AB
1898 */
1899#ifndef NET_SKB_PAD
5933dd2f 1900#define NET_SKB_PAD max(32, L1_CACHE_BYTES)
025be81e
AB
1901#endif
1902
7965bd4d 1903int ___pskb_trim(struct sk_buff *skb, unsigned int len);
1da177e4
LT
1904
1905static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
1906{
c4264f27 1907 if (unlikely(skb_is_nonlinear(skb))) {
3cc0e873
HX
1908 WARN_ON(1);
1909 return;
1910 }
27a884dc
ACM
1911 skb->len = len;
1912 skb_set_tail_pointer(skb, len);
1da177e4
LT
1913}
1914
7965bd4d 1915void skb_trim(struct sk_buff *skb, unsigned int len);
1da177e4
LT
1916
1917static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
1918{
3cc0e873
HX
1919 if (skb->data_len)
1920 return ___pskb_trim(skb, len);
1921 __skb_trim(skb, len);
1922 return 0;
1da177e4
LT
1923}
1924
1925static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
1926{
1927 return (len < skb->len) ? __pskb_trim(skb, len) : 0;
1928}
1929
e9fa4f7b
HX
1930/**
1931 * pskb_trim_unique - remove end from a paged unique (not cloned) buffer
1932 * @skb: buffer to alter
1933 * @len: new length
1934 *
1935 * This is identical to pskb_trim except that the caller knows that
1936 * the skb is not cloned so we should never get an error due to out-
1937 * of-memory.
1938 */
1939static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
1940{
1941 int err = pskb_trim(skb, len);
1942 BUG_ON(err);
1943}
1944
1da177e4
LT
1945/**
1946 * skb_orphan - orphan a buffer
1947 * @skb: buffer to orphan
1948 *
1949 * If a buffer currently has an owner then we call the owner's
1950 * destructor function and make the @skb unowned. The buffer continues
1951 * to exist but is no longer charged to its former owner.
1952 */
1953static inline void skb_orphan(struct sk_buff *skb)
1954{
c34a7612 1955 if (skb->destructor) {
1da177e4 1956 skb->destructor(skb);
c34a7612
ED
1957 skb->destructor = NULL;
1958 skb->sk = NULL;
376c7311
ED
1959 } else {
1960 BUG_ON(skb->sk);
c34a7612 1961 }
1da177e4
LT
1962}
1963
a353e0ce
MT
1964/**
1965 * skb_orphan_frags - orphan the frags contained in a buffer
1966 * @skb: buffer to orphan frags from
1967 * @gfp_mask: allocation mask for replacement pages
1968 *
1969 * For each frag in the SKB which needs a destructor (i.e. has an
1970 * owner) create a copy of that frag and release the original
1971 * page by calling the destructor.
1972 */
1973static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
1974{
1975 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY)))
1976 return 0;
1977 return skb_copy_ubufs(skb, gfp_mask);
1978}
1979
1da177e4
LT
1980/**
1981 * __skb_queue_purge - empty a list
1982 * @list: list to empty
1983 *
1984 * Delete all buffers on an &sk_buff list. Each buffer is removed from
1985 * the list and one reference dropped. This function does not take the
1986 * list lock and the caller must hold the relevant locks to use it.
1987 */
7965bd4d 1988void skb_queue_purge(struct sk_buff_head *list);
1da177e4
LT
1989static inline void __skb_queue_purge(struct sk_buff_head *list)
1990{
1991 struct sk_buff *skb;
1992 while ((skb = __skb_dequeue(list)) != NULL)
1993 kfree_skb(skb);
1994}
1995
e5e67305
AD
1996#define NETDEV_FRAG_PAGE_MAX_ORDER get_order(32768)
1997#define NETDEV_FRAG_PAGE_MAX_SIZE (PAGE_SIZE << NETDEV_FRAG_PAGE_MAX_ORDER)
1998#define NETDEV_PAGECNT_MAX_BIAS NETDEV_FRAG_PAGE_MAX_SIZE
1999
7965bd4d 2000void *netdev_alloc_frag(unsigned int fragsz);
1da177e4 2001
7965bd4d
JP
2002struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
2003 gfp_t gfp_mask);
8af27456
CH
2004
2005/**
2006 * netdev_alloc_skb - allocate an skbuff for rx on a specific device
2007 * @dev: network device to receive on
2008 * @length: length to allocate
2009 *
2010 * Allocate a new &sk_buff and assign it a usage count of one. The
2011 * buffer has unspecified headroom built in. Users should allocate
2012 * the headroom they think they need without accounting for the
2013 * built in space. The built in space is used for optimisations.
2014 *
2015 * %NULL is returned if there is no free memory. Although this function
2016 * allocates memory it can be called from an interrupt.
2017 */
2018static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
6f532612 2019 unsigned int length)
8af27456
CH
2020{
2021 return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
2022}
2023
6f532612
ED
2024/* legacy helper around __netdev_alloc_skb() */
2025static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
2026 gfp_t gfp_mask)
2027{
2028 return __netdev_alloc_skb(NULL, length, gfp_mask);
2029}
2030
2031/* legacy helper around netdev_alloc_skb() */
2032static inline struct sk_buff *dev_alloc_skb(unsigned int length)
2033{
2034 return netdev_alloc_skb(NULL, length);
2035}
2036
2037
4915a0de
ED
2038static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
2039 unsigned int length, gfp_t gfp)
61321bbd 2040{
4915a0de 2041 struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
61321bbd
ED
2042
2043 if (NET_IP_ALIGN && skb)
2044 skb_reserve(skb, NET_IP_ALIGN);
2045 return skb;
2046}
2047
4915a0de
ED
2048static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
2049 unsigned int length)
2050{
2051 return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
2052}
2053
bc6fc9fa
FF
2054/**
2055 * __skb_alloc_pages - allocate pages for ps-rx on a skb and preserve pfmemalloc data
0614002b
MG
2056 * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX
2057 * @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used
2058 * @order: size of the allocation
2059 *
2060 * Allocate a new page.
2061 *
2062 * %NULL is returned if there is no free memory.
2063*/
2064static inline struct page *__skb_alloc_pages(gfp_t gfp_mask,
2065 struct sk_buff *skb,
2066 unsigned int order)
2067{
2068 struct page *page;
2069
2070 gfp_mask |= __GFP_COLD;
2071
2072 if (!(gfp_mask & __GFP_NOMEMALLOC))
2073 gfp_mask |= __GFP_MEMALLOC;
2074
2075 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
2076 if (skb && page && page->pfmemalloc)
2077 skb->pfmemalloc = true;
2078
2079 return page;
2080}
2081
2082/**
2083 * __skb_alloc_page - allocate a page for ps-rx for a given skb and preserve pfmemalloc data
2084 * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX
2085 * @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used
2086 *
2087 * Allocate a new page.
2088 *
2089 * %NULL is returned if there is no free memory.
2090 */
2091static inline struct page *__skb_alloc_page(gfp_t gfp_mask,
2092 struct sk_buff *skb)
2093{
2094 return __skb_alloc_pages(gfp_mask, skb, 0);
2095}
2096
2097/**
2098 * skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
2099 * @page: The page that was allocated from skb_alloc_page
2100 * @skb: The skb that may need pfmemalloc set
2101 */
2102static inline void skb_propagate_pfmemalloc(struct page *page,
2103 struct sk_buff *skb)
2104{
2105 if (page && page->pfmemalloc)
2106 skb->pfmemalloc = true;
2107}
2108
131ea667 2109/**
e227867f 2110 * skb_frag_page - retrieve the page referred to by a paged fragment
131ea667
IC
2111 * @frag: the paged fragment
2112 *
2113 * Returns the &struct page associated with @frag.
2114 */
2115static inline struct page *skb_frag_page(const skb_frag_t *frag)
2116{
a8605c60 2117 return frag->page.p;
131ea667
IC
2118}
2119
2120/**
2121 * __skb_frag_ref - take an addition reference on a paged fragment.
2122 * @frag: the paged fragment
2123 *
2124 * Takes an additional reference on the paged fragment @frag.
2125 */
2126static inline void __skb_frag_ref(skb_frag_t *frag)
2127{
2128 get_page(skb_frag_page(frag));
2129}
2130
2131/**
2132 * skb_frag_ref - take an addition reference on a paged fragment of an skb.
2133 * @skb: the buffer
2134 * @f: the fragment offset.
2135 *
2136 * Takes an additional reference on the @f'th paged fragment of @skb.
2137 */
2138static inline void skb_frag_ref(struct sk_buff *skb, int f)
2139{
2140 __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
2141}
2142
2143/**
2144 * __skb_frag_unref - release a reference on a paged fragment.
2145 * @frag: the paged fragment
2146 *
2147 * Releases a reference on the paged fragment @frag.
2148 */
2149static inline void __skb_frag_unref(skb_frag_t *frag)
2150{
2151 put_page(skb_frag_page(frag));
2152}
2153
2154/**
2155 * skb_frag_unref - release a reference on a paged fragment of an skb.
2156 * @skb: the buffer
2157 * @f: the fragment offset
2158 *
2159 * Releases a reference on the @f'th paged fragment of @skb.
2160 */
2161static inline void skb_frag_unref(struct sk_buff *skb, int f)
2162{
2163 __skb_frag_unref(&skb_shinfo(skb)->frags[f]);
2164}
2165
2166/**
2167 * skb_frag_address - gets the address of the data contained in a paged fragment
2168 * @frag: the paged fragment buffer
2169 *
2170 * Returns the address of the data within @frag. The page must already
2171 * be mapped.
2172 */
2173static inline void *skb_frag_address(const skb_frag_t *frag)
2174{
2175 return page_address(skb_frag_page(frag)) + frag->page_offset;
2176}
2177
2178/**
2179 * skb_frag_address_safe - gets the address of the data contained in a paged fragment
2180 * @frag: the paged fragment buffer
2181 *
2182 * Returns the address of the data within @frag. Checks that the page
2183 * is mapped and returns %NULL otherwise.
2184 */
2185static inline void *skb_frag_address_safe(const skb_frag_t *frag)
2186{
2187 void *ptr = page_address(skb_frag_page(frag));
2188 if (unlikely(!ptr))
2189 return NULL;
2190
2191 return ptr + frag->page_offset;
2192}
2193
2194/**
2195 * __skb_frag_set_page - sets the page contained in a paged fragment
2196 * @frag: the paged fragment
2197 * @page: the page to set
2198 *
2199 * Sets the fragment @frag to contain @page.
2200 */
2201static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page)
2202{
a8605c60 2203 frag->page.p = page;
131ea667
IC
2204}
2205
2206/**
2207 * skb_frag_set_page - sets the page contained in a paged fragment of an skb
2208 * @skb: the buffer
2209 * @f: the fragment offset
2210 * @page: the page to set
2211 *
2212 * Sets the @f'th fragment of @skb to contain @page.
2213 */
2214static inline void skb_frag_set_page(struct sk_buff *skb, int f,
2215 struct page *page)
2216{
2217 __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page);
2218}
2219
400dfd3a
ED
2220bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
2221
131ea667
IC
2222/**
2223 * skb_frag_dma_map - maps a paged fragment via the DMA API
f83347df 2224 * @dev: the device to map the fragment to
131ea667
IC
2225 * @frag: the paged fragment to map
2226 * @offset: the offset within the fragment (starting at the
2227 * fragment's own offset)
2228 * @size: the number of bytes to map
f83347df 2229 * @dir: the direction of the mapping (%PCI_DMA_*)
131ea667
IC
2230 *
2231 * Maps the page associated with @frag to @device.
2232 */
2233static inline dma_addr_t skb_frag_dma_map(struct device *dev,
2234 const skb_frag_t *frag,
2235 size_t offset, size_t size,
2236 enum dma_data_direction dir)
2237{
2238 return dma_map_page(dev, skb_frag_page(frag),
2239 frag->page_offset + offset, size, dir);
2240}
2241
117632e6
ED
2242static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
2243 gfp_t gfp_mask)
2244{
2245 return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
2246}
2247
bad93e9d
OP
2248
2249static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
2250 gfp_t gfp_mask)
2251{
2252 return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
2253}
2254
2255
334a8132
PM
2256/**
2257 * skb_clone_writable - is the header of a clone writable
2258 * @skb: buffer to check
2259 * @len: length up to which to write
2260 *
2261 * Returns true if modifying the header part of the cloned buffer
2262 * does not requires the data to be copied.
2263 */
05bdd2f1 2264static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
334a8132
PM
2265{
2266 return !skb_header_cloned(skb) &&
2267 skb_headroom(skb) + len <= skb->hdr_len;
2268}
2269
d9cc2048
HX
2270static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
2271 int cloned)
2272{
2273 int delta = 0;
2274
d9cc2048
HX
2275 if (headroom > skb_headroom(skb))
2276 delta = headroom - skb_headroom(skb);
2277
2278 if (delta || cloned)
2279 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
2280 GFP_ATOMIC);
2281 return 0;
2282}
2283
1da177e4
LT
2284/**
2285 * skb_cow - copy header of skb when it is required
2286 * @skb: buffer to cow
2287 * @headroom: needed headroom
2288 *
2289 * If the skb passed lacks sufficient headroom or its data part
2290 * is shared, data is reallocated. If reallocation fails, an error
2291 * is returned and original skb is not changed.
2292 *
2293 * The result is skb with writable area skb->head...skb->tail
2294 * and at least @headroom of space at head.
2295 */
2296static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
2297{
d9cc2048
HX
2298 return __skb_cow(skb, headroom, skb_cloned(skb));
2299}
1da177e4 2300
d9cc2048
HX
2301/**
2302 * skb_cow_head - skb_cow but only making the head writable
2303 * @skb: buffer to cow
2304 * @headroom: needed headroom
2305 *
2306 * This function is identical to skb_cow except that we replace the
2307 * skb_cloned check by skb_header_cloned. It should be used when
2308 * you only need to push on some header and do not need to modify
2309 * the data.
2310 */
2311static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
2312{
2313 return __skb_cow(skb, headroom, skb_header_cloned(skb));
1da177e4
LT
2314}
2315
2316/**
2317 * skb_padto - pad an skbuff up to a minimal size
2318 * @skb: buffer to pad
2319 * @len: minimal length
2320 *
2321 * Pads up a buffer to ensure the trailing bytes exist and are
2322 * blanked. If the buffer already contains sufficient data it
5b057c6b
HX
2323 * is untouched. Otherwise it is extended. Returns zero on
2324 * success. The skb is freed on error.
1da177e4
LT
2325 */
2326
5b057c6b 2327static inline int skb_padto(struct sk_buff *skb, unsigned int len)
1da177e4
LT
2328{
2329 unsigned int size = skb->len;
2330 if (likely(size >= len))
5b057c6b 2331 return 0;
987c402a 2332 return skb_pad(skb, len - size);
1da177e4
LT
2333}
2334
2335static inline int skb_add_data(struct sk_buff *skb,
2336 char __user *from, int copy)
2337{
2338 const int off = skb->len;
2339
2340 if (skb->ip_summed == CHECKSUM_NONE) {
2341 int err = 0;
5084205f 2342 __wsum csum = csum_and_copy_from_user(from, skb_put(skb, copy),
1da177e4
LT
2343 copy, 0, &err);
2344 if (!err) {
2345 skb->csum = csum_block_add(skb->csum, csum, off);
2346 return 0;
2347 }
2348 } else if (!copy_from_user(skb_put(skb, copy), from, copy))
2349 return 0;
2350
2351 __skb_trim(skb, off);
2352 return -EFAULT;
2353}
2354
38ba0a65
ED
2355static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
2356 const struct page *page, int off)
1da177e4
LT
2357{
2358 if (i) {
9e903e08 2359 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1da177e4 2360
ea2ab693 2361 return page == skb_frag_page(frag) &&
9e903e08 2362 off == frag->page_offset + skb_frag_size(frag);
1da177e4 2363 }
38ba0a65 2364 return false;
1da177e4
LT
2365}
2366
364c6bad
HX
2367static inline int __skb_linearize(struct sk_buff *skb)
2368{
2369 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
2370}
2371
1da177e4
LT
2372/**
2373 * skb_linearize - convert paged skb to linear one
2374 * @skb: buffer to linarize
1da177e4
LT
2375 *
2376 * If there is no free memory -ENOMEM is returned, otherwise zero
2377 * is returned and the old skb data released.
2378 */
364c6bad
HX
2379static inline int skb_linearize(struct sk_buff *skb)
2380{
2381 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
2382}
2383
cef401de
ED
2384/**
2385 * skb_has_shared_frag - can any frag be overwritten
2386 * @skb: buffer to test
2387 *
2388 * Return true if the skb has at least one frag that might be modified
2389 * by an external entity (as in vmsplice()/sendfile())
2390 */
2391static inline bool skb_has_shared_frag(const struct sk_buff *skb)
2392{
c9af6db4
PS
2393 return skb_is_nonlinear(skb) &&
2394 skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
cef401de
ED
2395}
2396
364c6bad
HX
2397/**
2398 * skb_linearize_cow - make sure skb is linear and writable
2399 * @skb: buffer to process
2400 *
2401 * If there is no free memory -ENOMEM is returned, otherwise zero
2402 * is returned and the old skb data released.
2403 */
2404static inline int skb_linearize_cow(struct sk_buff *skb)
1da177e4 2405{
364c6bad
HX
2406 return skb_is_nonlinear(skb) || skb_cloned(skb) ?
2407 __skb_linearize(skb) : 0;
1da177e4
LT
2408}
2409
2410/**
2411 * skb_postpull_rcsum - update checksum for received skb after pull
2412 * @skb: buffer to update
2413 * @start: start of data before pull
2414 * @len: length of data pulled
2415 *
2416 * After doing a pull on a received packet, you need to call this to
84fa7933
PM
2417 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to
2418 * CHECKSUM_NONE so that it can be recomputed from scratch.
1da177e4
LT
2419 */
2420
2421static inline void skb_postpull_rcsum(struct sk_buff *skb,
cbb042f9 2422 const void *start, unsigned int len)
1da177e4 2423{
84fa7933 2424 if (skb->ip_summed == CHECKSUM_COMPLETE)
1da177e4
LT
2425 skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0));
2426}
2427
cbb042f9
HX
2428unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
2429
7ce5a27f
DM
2430/**
2431 * pskb_trim_rcsum - trim received skb and update checksum
2432 * @skb: buffer to trim
2433 * @len: new length
2434 *
2435 * This is exactly the same as pskb_trim except that it ensures the
2436 * checksum of received packets are still valid after the operation.
2437 */
2438
2439static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
2440{
2441 if (likely(len >= skb->len))
2442 return 0;
2443 if (skb->ip_summed == CHECKSUM_COMPLETE)
2444 skb->ip_summed = CHECKSUM_NONE;
2445 return __pskb_trim(skb, len);
2446}
2447
1da177e4
LT
2448#define skb_queue_walk(queue, skb) \
2449 for (skb = (queue)->next; \
a1e4891f 2450 skb != (struct sk_buff *)(queue); \
1da177e4
LT
2451 skb = skb->next)
2452
46f8914e
JC
2453#define skb_queue_walk_safe(queue, skb, tmp) \
2454 for (skb = (queue)->next, tmp = skb->next; \
2455 skb != (struct sk_buff *)(queue); \
2456 skb = tmp, tmp = skb->next)
2457
1164f52a 2458#define skb_queue_walk_from(queue, skb) \
a1e4891f 2459 for (; skb != (struct sk_buff *)(queue); \
1164f52a
DM
2460 skb = skb->next)
2461
2462#define skb_queue_walk_from_safe(queue, skb, tmp) \
2463 for (tmp = skb->next; \
2464 skb != (struct sk_buff *)(queue); \
2465 skb = tmp, tmp = skb->next)
2466
300ce174
SH
2467#define skb_queue_reverse_walk(queue, skb) \
2468 for (skb = (queue)->prev; \
a1e4891f 2469 skb != (struct sk_buff *)(queue); \
300ce174
SH
2470 skb = skb->prev)
2471
686a2955
DM
2472#define skb_queue_reverse_walk_safe(queue, skb, tmp) \
2473 for (skb = (queue)->prev, tmp = skb->prev; \
2474 skb != (struct sk_buff *)(queue); \
2475 skb = tmp, tmp = skb->prev)
2476
2477#define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \
2478 for (tmp = skb->prev; \
2479 skb != (struct sk_buff *)(queue); \
2480 skb = tmp, tmp = skb->prev)
1da177e4 2481
21dc3301 2482static inline bool skb_has_frag_list(const struct sk_buff *skb)
ee039871
DM
2483{
2484 return skb_shinfo(skb)->frag_list != NULL;
2485}
2486
2487static inline void skb_frag_list_init(struct sk_buff *skb)
2488{
2489 skb_shinfo(skb)->frag_list = NULL;
2490}
2491
2492static inline void skb_frag_add_head(struct sk_buff *skb, struct sk_buff *frag)
2493{
2494 frag->next = skb_shinfo(skb)->frag_list;
2495 skb_shinfo(skb)->frag_list = frag;
2496}
2497
2498#define skb_walk_frags(skb, iter) \
2499 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
2500
7965bd4d
JP
2501struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
2502 int *peeked, int *off, int *err);
2503struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, int noblock,
2504 int *err);
2505unsigned int datagram_poll(struct file *file, struct socket *sock,
2506 struct poll_table_struct *wait);
2507int skb_copy_datagram_iovec(const struct sk_buff *from, int offset,
2508 struct iovec *to, int size);
2509int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb, int hlen,
2510 struct iovec *iov);
2511int skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset,
2512 const struct iovec *from, int from_offset,
2513 int len);
2514int zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *frm,
2515 int offset, size_t count);
2516int skb_copy_datagram_const_iovec(const struct sk_buff *from, int offset,
2517 const struct iovec *to, int to_offset,
2518 int size);
2519void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
2520void skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb);
2521int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
7965bd4d
JP
2522int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
2523int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
2524__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
2525 int len, __wsum csum);
2526int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
2527 struct pipe_inode_info *pipe, unsigned int len,
2528 unsigned int flags);
2529void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
af2806f8 2530unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
36d5fe6a
ZK
2531int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
2532 int len, int hlen);
7965bd4d
JP
2533void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
2534int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
2535void skb_scrub_packet(struct sk_buff *skb, bool xnet);
de960aa9 2536unsigned int skb_gso_transport_seglen(const struct sk_buff *skb);
7965bd4d 2537struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
20380731 2538
2817a336
DB
2539struct skb_checksum_ops {
2540 __wsum (*update)(const void *mem, int len, __wsum wsum);
2541 __wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
2542};
2543
2544__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2545 __wsum csum, const struct skb_checksum_ops *ops);
2546__wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
2547 __wsum csum);
2548
1da177e4
LT
2549static inline void *skb_header_pointer(const struct sk_buff *skb, int offset,
2550 int len, void *buffer)
2551{
2552 int hlen = skb_headlen(skb);
2553
55820ee2 2554 if (hlen - offset >= len)
1da177e4
LT
2555 return skb->data + offset;
2556
2557 if (skb_copy_bits(skb, offset, buffer, len) < 0)
2558 return NULL;
2559
2560 return buffer;
2561}
2562
4262e5cc
DB
2563/**
2564 * skb_needs_linearize - check if we need to linearize a given skb
2565 * depending on the given device features.
2566 * @skb: socket buffer to check
2567 * @features: net device features
2568 *
2569 * Returns true if either:
2570 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2571 * 2. skb is fragmented and the device does not support SG.
2572 */
2573static inline bool skb_needs_linearize(struct sk_buff *skb,
2574 netdev_features_t features)
2575{
2576 return skb_is_nonlinear(skb) &&
2577 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
2578 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
2579}
2580
d626f62b
ACM
2581static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
2582 void *to,
2583 const unsigned int len)
2584{
2585 memcpy(to, skb->data, len);
2586}
2587
2588static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
2589 const int offset, void *to,
2590 const unsigned int len)
2591{
2592 memcpy(to, skb->data + offset, len);
2593}
2594
27d7ff46
ACM
2595static inline void skb_copy_to_linear_data(struct sk_buff *skb,
2596 const void *from,
2597 const unsigned int len)
2598{
2599 memcpy(skb->data, from, len);
2600}
2601
2602static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
2603 const int offset,
2604 const void *from,
2605 const unsigned int len)
2606{
2607 memcpy(skb->data + offset, from, len);
2608}
2609
7965bd4d 2610void skb_init(void);
1da177e4 2611
ac45f602
PO
2612static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
2613{
2614 return skb->tstamp;
2615}
2616
a61bbcf2
PM
2617/**
2618 * skb_get_timestamp - get timestamp from a skb
2619 * @skb: skb to get stamp from
2620 * @stamp: pointer to struct timeval to store stamp in
2621 *
2622 * Timestamps are stored in the skb as offsets to a base timestamp.
2623 * This function converts the offset back to a struct timeval and stores
2624 * it in stamp.
2625 */
ac45f602
PO
2626static inline void skb_get_timestamp(const struct sk_buff *skb,
2627 struct timeval *stamp)
a61bbcf2 2628{
b7aa0bf7 2629 *stamp = ktime_to_timeval(skb->tstamp);
a61bbcf2
PM
2630}
2631
ac45f602
PO
2632static inline void skb_get_timestampns(const struct sk_buff *skb,
2633 struct timespec *stamp)
2634{
2635 *stamp = ktime_to_timespec(skb->tstamp);
2636}
2637
b7aa0bf7 2638static inline void __net_timestamp(struct sk_buff *skb)
a61bbcf2 2639{
b7aa0bf7 2640 skb->tstamp = ktime_get_real();
a61bbcf2
PM
2641}
2642
164891aa
SH
2643static inline ktime_t net_timedelta(ktime_t t)
2644{
2645 return ktime_sub(ktime_get_real(), t);
2646}
2647
b9ce204f
IJ
2648static inline ktime_t net_invalid_timestamp(void)
2649{
2650 return ktime_set(0, 0);
2651}
a61bbcf2 2652
c1f19b51
RC
2653#ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
2654
7965bd4d
JP
2655void skb_clone_tx_timestamp(struct sk_buff *skb);
2656bool skb_defer_rx_timestamp(struct sk_buff *skb);
c1f19b51
RC
2657
2658#else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
2659
2660static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
2661{
2662}
2663
2664static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
2665{
2666 return false;
2667}
2668
2669#endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
2670
2671/**
2672 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
2673 *
da92b194
RC
2674 * PHY drivers may accept clones of transmitted packets for
2675 * timestamping via their phy_driver.txtstamp method. These drivers
2676 * must call this function to return the skb back to the stack, with
2677 * or without a timestamp.
2678 *
c1f19b51 2679 * @skb: clone of the the original outgoing packet
da92b194 2680 * @hwtstamps: hardware time stamps, may be NULL if not available
c1f19b51
RC
2681 *
2682 */
2683void skb_complete_tx_timestamp(struct sk_buff *skb,
2684 struct skb_shared_hwtstamps *hwtstamps);
2685
ac45f602
PO
2686/**
2687 * skb_tstamp_tx - queue clone of skb with send time stamps
2688 * @orig_skb: the original outgoing packet
2689 * @hwtstamps: hardware time stamps, may be NULL if not available
2690 *
2691 * If the skb has a socket associated, then this function clones the
2692 * skb (thus sharing the actual data and optional structures), stores
2693 * the optional hardware time stamping information (if non NULL) or
2694 * generates a software time stamp (otherwise), then queues the clone
2695 * to the error queue of the socket. Errors are silently ignored.
2696 */
7965bd4d
JP
2697void skb_tstamp_tx(struct sk_buff *orig_skb,
2698 struct skb_shared_hwtstamps *hwtstamps);
ac45f602 2699
4507a715
RC
2700static inline void sw_tx_timestamp(struct sk_buff *skb)
2701{
2244d07b
OH
2702 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP &&
2703 !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
4507a715
RC
2704 skb_tstamp_tx(skb, NULL);
2705}
2706
2707/**
2708 * skb_tx_timestamp() - Driver hook for transmit timestamping
2709 *
2710 * Ethernet MAC Drivers should call this function in their hard_xmit()
4ff75b7c 2711 * function immediately before giving the sk_buff to the MAC hardware.
4507a715 2712 *
73409f3b
DM
2713 * Specifically, one should make absolutely sure that this function is
2714 * called before TX completion of this packet can trigger. Otherwise
2715 * the packet could potentially already be freed.
2716 *
4507a715
RC
2717 * @skb: A socket buffer.
2718 */
2719static inline void skb_tx_timestamp(struct sk_buff *skb)
2720{
c1f19b51 2721 skb_clone_tx_timestamp(skb);
4507a715
RC
2722 sw_tx_timestamp(skb);
2723}
2724
6e3e939f
JB
2725/**
2726 * skb_complete_wifi_ack - deliver skb with wifi status
2727 *
2728 * @skb: the original outgoing packet
2729 * @acked: ack status
2730 *
2731 */
2732void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
2733
7965bd4d
JP
2734__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
2735__sum16 __skb_checksum_complete(struct sk_buff *skb);
fb286bb2 2736
60476372
HX
2737static inline int skb_csum_unnecessary(const struct sk_buff *skb)
2738{
5d0c2b95 2739 return ((skb->ip_summed & CHECKSUM_UNNECESSARY) || skb->csum_valid);
60476372
HX
2740}
2741
fb286bb2
HX
2742/**
2743 * skb_checksum_complete - Calculate checksum of an entire packet
2744 * @skb: packet to process
2745 *
2746 * This function calculates the checksum over the entire packet plus
2747 * the value of skb->csum. The latter can be used to supply the
2748 * checksum of a pseudo header as used by TCP/UDP. It returns the
2749 * checksum.
2750 *
2751 * For protocols that contain complete checksums such as ICMP/TCP/UDP,
2752 * this function can be used to verify that checksum on received
2753 * packets. In that case the function should return zero if the
2754 * checksum is correct. In particular, this function will return zero
2755 * if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
2756 * hardware has already verified the correctness of the checksum.
2757 */
4381ca3c 2758static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
fb286bb2 2759{
60476372
HX
2760 return skb_csum_unnecessary(skb) ?
2761 0 : __skb_checksum_complete(skb);
fb286bb2
HX
2762}
2763
76ba0aae
TH
2764/* Check if we need to perform checksum complete validation.
2765 *
2766 * Returns true if checksum complete is needed, false otherwise
2767 * (either checksum is unnecessary or zero checksum is allowed).
2768 */
2769static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
2770 bool zero_okay,
2771 __sum16 check)
2772{
5d0c2b95
TH
2773 if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
2774 skb->csum_valid = 1;
76ba0aae
TH
2775 return false;
2776 }
2777
2778 return true;
2779}
2780
2781/* For small packets <= CHECKSUM_BREAK peform checksum complete directly
2782 * in checksum_init.
2783 */
2784#define CHECKSUM_BREAK 76
2785
2786/* Validate (init) checksum based on checksum complete.
2787 *
2788 * Return values:
2789 * 0: checksum is validated or try to in skb_checksum_complete. In the latter
2790 * case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
2791 * checksum is stored in skb->csum for use in __skb_checksum_complete
2792 * non-zero: value of invalid checksum
2793 *
2794 */
2795static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
2796 bool complete,
2797 __wsum psum)
2798{
2799 if (skb->ip_summed == CHECKSUM_COMPLETE) {
2800 if (!csum_fold(csum_add(psum, skb->csum))) {
5d0c2b95 2801 skb->csum_valid = 1;
76ba0aae
TH
2802 return 0;
2803 }
2804 }
2805
2806 skb->csum = psum;
2807
5d0c2b95
TH
2808 if (complete || skb->len <= CHECKSUM_BREAK) {
2809 __sum16 csum;
2810
2811 csum = __skb_checksum_complete(skb);
2812 skb->csum_valid = !csum;
2813 return csum;
2814 }
76ba0aae
TH
2815
2816 return 0;
2817}
2818
2819static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
2820{
2821 return 0;
2822}
2823
2824/* Perform checksum validate (init). Note that this is a macro since we only
2825 * want to calculate the pseudo header which is an input function if necessary.
2826 * First we try to validate without any computation (checksum unnecessary) and
2827 * then calculate based on checksum complete calling the function to compute
2828 * pseudo header.
2829 *
2830 * Return values:
2831 * 0: checksum is validated or try to in skb_checksum_complete
2832 * non-zero: value of invalid checksum
2833 */
2834#define __skb_checksum_validate(skb, proto, complete, \
2835 zero_okay, check, compute_pseudo) \
2836({ \
2837 __sum16 __ret = 0; \
5d0c2b95 2838 skb->csum_valid = 0; \
76ba0aae
TH
2839 if (__skb_checksum_validate_needed(skb, zero_okay, check)) \
2840 __ret = __skb_checksum_validate_complete(skb, \
2841 complete, compute_pseudo(skb, proto)); \
2842 __ret; \
2843})
2844
2845#define skb_checksum_init(skb, proto, compute_pseudo) \
2846 __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
2847
2848#define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \
2849 __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
2850
2851#define skb_checksum_validate(skb, proto, compute_pseudo) \
2852 __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
2853
2854#define skb_checksum_validate_zero_check(skb, proto, check, \
2855 compute_pseudo) \
2856 __skb_checksum_validate_(skb, proto, true, true, check, compute_pseudo)
2857
2858#define skb_checksum_simple_validate(skb) \
2859 __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
2860
5f79e0f9 2861#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
7965bd4d 2862void nf_conntrack_destroy(struct nf_conntrack *nfct);
1da177e4
LT
2863static inline void nf_conntrack_put(struct nf_conntrack *nfct)
2864{
2865 if (nfct && atomic_dec_and_test(&nfct->use))
de6e05c4 2866 nf_conntrack_destroy(nfct);
1da177e4
LT
2867}
2868static inline void nf_conntrack_get(struct nf_conntrack *nfct)
2869{
2870 if (nfct)
2871 atomic_inc(&nfct->use);
2872}
2fc72c7b 2873#endif
1da177e4
LT
2874#ifdef CONFIG_BRIDGE_NETFILTER
2875static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge)
2876{
2877 if (nf_bridge && atomic_dec_and_test(&nf_bridge->use))
2878 kfree(nf_bridge);
2879}
2880static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
2881{
2882 if (nf_bridge)
2883 atomic_inc(&nf_bridge->use);
2884}
2885#endif /* CONFIG_BRIDGE_NETFILTER */
a193a4ab
PM
2886static inline void nf_reset(struct sk_buff *skb)
2887{
5f79e0f9 2888#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
a193a4ab
PM
2889 nf_conntrack_put(skb->nfct);
2890 skb->nfct = NULL;
2fc72c7b 2891#endif
a193a4ab
PM
2892#ifdef CONFIG_BRIDGE_NETFILTER
2893 nf_bridge_put(skb->nf_bridge);
2894 skb->nf_bridge = NULL;
2895#endif
2896}
2897
124dff01
PM
2898static inline void nf_reset_trace(struct sk_buff *skb)
2899{
478b360a 2900#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
130549fe
G
2901 skb->nf_trace = 0;
2902#endif
a193a4ab
PM
2903}
2904
edda553c
YK
2905/* Note: This doesn't put any conntrack and bridge info in dst. */
2906static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2907{
5f79e0f9 2908#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
edda553c
YK
2909 dst->nfct = src->nfct;
2910 nf_conntrack_get(src->nfct);
2911 dst->nfctinfo = src->nfctinfo;
2fc72c7b 2912#endif
edda553c
YK
2913#ifdef CONFIG_BRIDGE_NETFILTER
2914 dst->nf_bridge = src->nf_bridge;
2915 nf_bridge_get(src->nf_bridge);
2916#endif
478b360a
FW
2917#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
2918 dst->nf_trace = src->nf_trace;
2919#endif
edda553c
YK
2920}
2921
e7ac05f3
YK
2922static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2923{
e7ac05f3 2924#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
5f79e0f9 2925 nf_conntrack_put(dst->nfct);
2fc72c7b 2926#endif
e7ac05f3
YK
2927#ifdef CONFIG_BRIDGE_NETFILTER
2928 nf_bridge_put(dst->nf_bridge);
2929#endif
2930 __nf_copy(dst, src);
2931}
2932
984bc16c
JM
2933#ifdef CONFIG_NETWORK_SECMARK
2934static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2935{
2936 to->secmark = from->secmark;
2937}
2938
2939static inline void skb_init_secmark(struct sk_buff *skb)
2940{
2941 skb->secmark = 0;
2942}
2943#else
2944static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2945{ }
2946
2947static inline void skb_init_secmark(struct sk_buff *skb)
2948{ }
2949#endif
2950
574f7194
EB
2951static inline bool skb_irq_freeable(const struct sk_buff *skb)
2952{
2953 return !skb->destructor &&
2954#if IS_ENABLED(CONFIG_XFRM)
2955 !skb->sp &&
2956#endif
2957#if IS_ENABLED(CONFIG_NF_CONNTRACK)
2958 !skb->nfct &&
2959#endif
2960 !skb->_skb_refdst &&
2961 !skb_has_frag_list(skb);
2962}
2963
f25f4e44
PWJ
2964static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
2965{
f25f4e44 2966 skb->queue_mapping = queue_mapping;
f25f4e44
PWJ
2967}
2968
9247744e 2969static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
4e3ab47a 2970{
4e3ab47a 2971 return skb->queue_mapping;
4e3ab47a
PE
2972}
2973
f25f4e44
PWJ
2974static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
2975{
f25f4e44 2976 to->queue_mapping = from->queue_mapping;
f25f4e44
PWJ
2977}
2978
d5a9e24a
DM
2979static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
2980{
2981 skb->queue_mapping = rx_queue + 1;
2982}
2983
9247744e 2984static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
d5a9e24a
DM
2985{
2986 return skb->queue_mapping - 1;
2987}
2988
9247744e 2989static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
d5a9e24a 2990{
a02cec21 2991 return skb->queue_mapping != 0;
d5a9e24a
DM
2992}
2993
7965bd4d
JP
2994u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2995 unsigned int num_tx_queues);
9247744e 2996
def8b4fa
AD
2997static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
2998{
0b3d8e08 2999#ifdef CONFIG_XFRM
def8b4fa 3000 return skb->sp;
def8b4fa 3001#else
def8b4fa 3002 return NULL;
def8b4fa 3003#endif
0b3d8e08 3004}
def8b4fa 3005
68c33163
PS
3006/* Keeps track of mac header offset relative to skb->head.
3007 * It is useful for TSO of Tunneling protocol. e.g. GRE.
3008 * For non-tunnel skb it points to skb_mac_header() and for
3347c960
ED
3009 * tunnel skb it points to outer mac header.
3010 * Keeps track of level of encapsulation of network headers.
3011 */
68c33163 3012struct skb_gso_cb {
3347c960
ED
3013 int mac_offset;
3014 int encap_level;
7e2b10c1 3015 __u16 csum_start;
68c33163
PS
3016};
3017#define SKB_GSO_CB(skb) ((struct skb_gso_cb *)(skb)->cb)
3018
3019static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
3020{
3021 return (skb_mac_header(inner_skb) - inner_skb->head) -
3022 SKB_GSO_CB(inner_skb)->mac_offset;
3023}
3024
1e2bd517
PS
3025static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra)
3026{
3027 int new_headroom, headroom;
3028 int ret;
3029
3030 headroom = skb_headroom(skb);
3031 ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC);
3032 if (ret)
3033 return ret;
3034
3035 new_headroom = skb_headroom(skb);
3036 SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom);
3037 return 0;
3038}
3039
7e2b10c1
TH
3040/* Compute the checksum for a gso segment. First compute the checksum value
3041 * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and
3042 * then add in skb->csum (checksum from csum_start to end of packet).
3043 * skb->csum and csum_start are then updated to reflect the checksum of the
3044 * resultant packet starting from the transport header-- the resultant checksum
3045 * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo
3046 * header.
3047 */
3048static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res)
3049{
3050 int plen = SKB_GSO_CB(skb)->csum_start - skb_headroom(skb) -
3051 skb_transport_offset(skb);
3052 __u16 csum;
3053
3054 csum = csum_fold(csum_partial(skb_transport_header(skb),
3055 plen, skb->csum));
3056 skb->csum = res;
3057 SKB_GSO_CB(skb)->csum_start -= plen;
3058
3059 return csum;
3060}
3061
bdcc0924 3062static inline bool skb_is_gso(const struct sk_buff *skb)
89114afd
HX
3063{
3064 return skb_shinfo(skb)->gso_size;
3065}
3066
36a8f39e 3067/* Note: Should be called only if skb_is_gso(skb) is true */
bdcc0924 3068static inline bool skb_is_gso_v6(const struct sk_buff *skb)
eabd7e35
BG
3069{
3070 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
3071}
3072
7965bd4d 3073void __skb_warn_lro_forwarding(const struct sk_buff *skb);
4497b076
BH
3074
3075static inline bool skb_warn_if_lro(const struct sk_buff *skb)
3076{
3077 /* LRO sets gso_size but not gso_type, whereas if GSO is really
3078 * wanted then gso_type will be set. */
05bdd2f1
ED
3079 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3080
b78462eb
AD
3081 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
3082 unlikely(shinfo->gso_type == 0)) {
4497b076
BH
3083 __skb_warn_lro_forwarding(skb);
3084 return true;
3085 }
3086 return false;
3087}
3088
35fc92a9
HX
3089static inline void skb_forward_csum(struct sk_buff *skb)
3090{
3091 /* Unfortunately we don't support this one. Any brave souls? */
3092 if (skb->ip_summed == CHECKSUM_COMPLETE)
3093 skb->ip_summed = CHECKSUM_NONE;
3094}
3095
bc8acf2c
ED
3096/**
3097 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
3098 * @skb: skb to check
3099 *
3100 * fresh skbs have their ip_summed set to CHECKSUM_NONE.
3101 * Instead of forcing ip_summed to CHECKSUM_NONE, we can
3102 * use this helper, to document places where we make this assertion.
3103 */
05bdd2f1 3104static inline void skb_checksum_none_assert(const struct sk_buff *skb)
bc8acf2c
ED
3105{
3106#ifdef DEBUG
3107 BUG_ON(skb->ip_summed != CHECKSUM_NONE);
3108#endif
3109}
3110
f35d9d8a 3111bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
a6686f2f 3112
ed1f50c3
PD
3113int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
3114
f77668dc
DB
3115u32 __skb_get_poff(const struct sk_buff *skb);
3116
3a7c1ee4
AD
3117/**
3118 * skb_head_is_locked - Determine if the skb->head is locked down
3119 * @skb: skb to check
3120 *
3121 * The head on skbs build around a head frag can be removed if they are
3122 * not cloned. This function returns true if the skb head is locked down
3123 * due to either being allocated via kmalloc, or by being a clone with
3124 * multiple references to the head.
3125 */
3126static inline bool skb_head_is_locked(const struct sk_buff *skb)
3127{
3128 return !skb->head_frag || skb_cloned(skb);
3129}
fe6cc55f
FW
3130
3131/**
3132 * skb_gso_network_seglen - Return length of individual segments of a gso packet
3133 *
3134 * @skb: GSO skb
3135 *
3136 * skb_gso_network_seglen is used to determine the real size of the
3137 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
3138 *
3139 * The MAC/L2 header is not accounted for.
3140 */
3141static inline unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
3142{
3143 unsigned int hdr_len = skb_transport_header(skb) -
3144 skb_network_header(skb);
3145 return hdr_len + skb_gso_transport_seglen(skb);
3146}
1da177e4
LT
3147#endif /* __KERNEL__ */
3148#endif /* _LINUX_SKBUFF_H */