1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
7 * Definitions for the UDP module.
9 * Version: @(#)udp.h 1.0.2 05/07/93
12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
15 * Alan Cox : Turned on udp checksums. I don't want to
16 * chase 'memory corruption' bugs that aren't!
21 #include <linux/list.h>
22 #include <linux/bug.h>
23 #include <net/inet_sock.h>
27 #include <linux/ipv6.h>
28 #include <linux/seq_file.h>
29 #include <linux/poll.h>
30 #include <linux/indirect_call_wrapper.h>
33 * struct udp_skb_cb - UDP(-Lite) private variables
35 * @header: private variables used by IPv4/IPv6
36 * @cscov: checksum coverage length (UDP-Lite only)
37 * @partial_cov: if set indicates partial csum coverage
41 struct inet_skb_parm h4
;
42 #if IS_ENABLED(CONFIG_IPV6)
43 struct inet6_skb_parm h6
;
49 #define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb))
52 * struct udp_hslot - UDP hash slot
54 * @head: head of list of sockets
55 * @count: number of sockets in 'head' list
56 * @lock: spinlock protecting changes to head/count
59 struct hlist_head head
;
62 } __attribute__((aligned(2 * sizeof(long))));
65 * struct udp_table - UDP table
67 * @hash: hash table, sockets are hashed on (local port)
68 * @hash2: hash table, sockets are hashed on (local port, local address)
69 * @mask: number of slots in hash tables, minus 1
70 * @log: log2(number of slots in hash table)
73 struct udp_hslot
*hash
;
74 struct udp_hslot
*hash2
;
78 extern struct udp_table udp_table
;
79 void udp_table_init(struct udp_table
*, const char *);
80 static inline struct udp_hslot
*udp_hashslot(struct udp_table
*table
,
81 struct net
*net
, unsigned int num
)
83 return &table
->hash
[udp_hashfn(net
, num
, table
->mask
)];
86 * For secondary hash, net_hash_mix() is performed before calling
87 * udp_hashslot2(), this explains difference with udp_hashslot()
89 static inline struct udp_hslot
*udp_hashslot2(struct udp_table
*table
,
92 return &table
->hash2
[hash
& table
->mask
];
95 extern struct proto udp_prot
;
97 extern atomic_long_t udp_memory_allocated
;
99 /* sysctl variables for udp */
100 extern long sysctl_udp_mem
[3];
101 extern int sysctl_udp_rmem_min
;
102 extern int sysctl_udp_wmem_min
;
107 * Generic checksumming routines for UDP(-Lite) v4 and v6
109 static inline __sum16
__udp_lib_checksum_complete(struct sk_buff
*skb
)
111 return (UDP_SKB_CB(skb
)->cscov
== skb
->len
?
112 __skb_checksum_complete(skb
) :
113 __skb_checksum_complete_head(skb
, UDP_SKB_CB(skb
)->cscov
));
116 static inline int udp_lib_checksum_complete(struct sk_buff
*skb
)
118 return !skb_csum_unnecessary(skb
) &&
119 __udp_lib_checksum_complete(skb
);
123 * udp_csum_outgoing - compute UDPv4/v6 checksum over fragments
124 * @sk: socket we are writing to
125 * @skb: sk_buff containing the filled-in UDP header
126 * (checksum field must be zeroed out)
128 static inline __wsum
udp_csum_outgoing(struct sock
*sk
, struct sk_buff
*skb
)
130 __wsum csum
= csum_partial(skb_transport_header(skb
),
131 sizeof(struct udphdr
), 0);
132 skb_queue_walk(&sk
->sk_write_queue
, skb
) {
133 csum
= csum_add(csum
, skb
->csum
);
138 static inline __wsum
udp_csum(struct sk_buff
*skb
)
140 __wsum csum
= csum_partial(skb_transport_header(skb
),
141 sizeof(struct udphdr
), skb
->csum
);
143 for (skb
= skb_shinfo(skb
)->frag_list
; skb
; skb
= skb
->next
) {
144 csum
= csum_add(csum
, skb
->csum
);
149 static inline __sum16
udp_v4_check(int len
, __be32 saddr
,
150 __be32 daddr
, __wsum base
)
152 return csum_tcpudp_magic(saddr
, daddr
, len
, IPPROTO_UDP
, base
);
155 void udp_set_csum(bool nocheck
, struct sk_buff
*skb
,
156 __be32 saddr
, __be32 daddr
, int len
);
158 static inline void udp_csum_pull_header(struct sk_buff
*skb
)
160 if (!skb
->csum_valid
&& skb
->ip_summed
== CHECKSUM_NONE
)
161 skb
->csum
= csum_partial(skb
->data
, sizeof(struct udphdr
),
163 skb_pull_rcsum(skb
, sizeof(struct udphdr
));
164 UDP_SKB_CB(skb
)->cscov
-= sizeof(struct udphdr
);
167 typedef struct sock
*(*udp_lookup_t
)(const struct sk_buff
*skb
, __be16 sport
,
170 INDIRECT_CALLABLE_DECLARE(struct sk_buff
*udp4_gro_receive(struct list_head
*,
172 INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff
*, int));
173 INDIRECT_CALLABLE_DECLARE(struct sk_buff
*udp6_gro_receive(struct list_head
*,
175 INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff
*, int));
176 struct sk_buff
*udp_gro_receive(struct list_head
*head
, struct sk_buff
*skb
,
177 struct udphdr
*uh
, struct sock
*sk
);
178 int udp_gro_complete(struct sk_buff
*skb
, int nhoff
, udp_lookup_t lookup
);
180 struct sk_buff
*__udp_gso_segment(struct sk_buff
*gso_skb
,
181 netdev_features_t features
);
183 static inline struct udphdr
*udp_gro_udphdr(struct sk_buff
*skb
)
186 unsigned int hlen
, off
;
188 off
= skb_gro_offset(skb
);
189 hlen
= off
+ sizeof(*uh
);
190 uh
= skb_gro_header_fast(skb
, off
);
191 if (skb_gro_header_hard(skb
, hlen
))
192 uh
= skb_gro_header_slow(skb
, hlen
, off
);
197 /* hash routines shared between UDPv4/6 and UDP-Litev4/6 */
198 static inline int udp_lib_hash(struct sock
*sk
)
204 void udp_lib_unhash(struct sock
*sk
);
205 void udp_lib_rehash(struct sock
*sk
, u16 new_hash
);
207 static inline void udp_lib_close(struct sock
*sk
, long timeout
)
209 sk_common_release(sk
);
212 int udp_lib_get_port(struct sock
*sk
, unsigned short snum
,
213 unsigned int hash2_nulladdr
);
215 u32
udp_flow_hashrnd(void);
217 static inline __be16
udp_flow_src_port(struct net
*net
, struct sk_buff
*skb
,
218 int min
, int max
, bool use_eth
)
223 /* Use default range */
224 inet_get_local_port_range(net
, &min
, &max
);
227 hash
= skb_get_hash(skb
);
228 if (unlikely(!hash
)) {
230 /* Can't find a normal hash, caller has indicated an
231 * Ethernet packet so use that to compute a hash.
233 hash
= jhash(skb
->data
, 2 * ETH_ALEN
,
234 (__force u32
) skb
->protocol
);
236 /* Can't derive any sort of hash for the packet, set
237 * to some consistent random value.
239 hash
= udp_flow_hashrnd();
243 /* Since this is being sent on the wire obfuscate hash a bit
244 * to minimize possbility that any useful information to an
245 * attacker is leaked. Only upper 16 bits are relevant in the
246 * computation for 16 bit port value.
250 return htons((((u64
) hash
* (max
- min
)) >> 32) + min
);
253 static inline int udp_rqueue_get(struct sock
*sk
)
255 return sk_rmem_alloc_get(sk
) - READ_ONCE(udp_sk(sk
)->forward_deficit
);
258 static inline bool udp_sk_bound_dev_eq(struct net
*net
, int bound_dev_if
,
261 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
262 return inet_bound_dev_eq(!!net
->ipv4
.sysctl_udp_l3mdev_accept
,
263 bound_dev_if
, dif
, sdif
);
265 return inet_bound_dev_eq(true, bound_dev_if
, dif
, sdif
);
270 void udp_destruct_sock(struct sock
*sk
);
271 void skb_consume_udp(struct sock
*sk
, struct sk_buff
*skb
, int len
);
272 int __udp_enqueue_schedule_skb(struct sock
*sk
, struct sk_buff
*skb
);
273 void udp_skb_destructor(struct sock
*sk
, struct sk_buff
*skb
);
274 struct sk_buff
*__skb_recv_udp(struct sock
*sk
, unsigned int flags
,
275 int noblock
, int *off
, int *err
);
276 static inline struct sk_buff
*skb_recv_udp(struct sock
*sk
, unsigned int flags
,
277 int noblock
, int *err
)
281 return __skb_recv_udp(sk
, flags
, noblock
, &off
, err
);
284 int udp_v4_early_demux(struct sk_buff
*skb
);
285 bool udp_sk_rx_dst_set(struct sock
*sk
, struct dst_entry
*dst
);
286 int udp_get_port(struct sock
*sk
, unsigned short snum
,
287 int (*saddr_cmp
)(const struct sock
*,
288 const struct sock
*));
289 int udp_err(struct sk_buff
*, u32
);
290 int udp_abort(struct sock
*sk
, int err
);
291 int udp_sendmsg(struct sock
*sk
, struct msghdr
*msg
, size_t len
);
292 int udp_push_pending_frames(struct sock
*sk
);
293 void udp_flush_pending_frames(struct sock
*sk
);
294 int udp_cmsg_send(struct sock
*sk
, struct msghdr
*msg
, u16
*gso_size
);
295 void udp4_hwcsum(struct sk_buff
*skb
, __be32 src
, __be32 dst
);
296 int udp_rcv(struct sk_buff
*skb
);
297 int udp_ioctl(struct sock
*sk
, int cmd
, unsigned long arg
);
298 int udp_init_sock(struct sock
*sk
);
299 int udp_pre_connect(struct sock
*sk
, struct sockaddr
*uaddr
, int addr_len
);
300 int __udp_disconnect(struct sock
*sk
, int flags
);
301 int udp_disconnect(struct sock
*sk
, int flags
);
302 __poll_t
udp_poll(struct file
*file
, struct socket
*sock
, poll_table
*wait
);
303 struct sk_buff
*skb_udp_tunnel_segment(struct sk_buff
*skb
,
304 netdev_features_t features
,
306 int udp_lib_getsockopt(struct sock
*sk
, int level
, int optname
,
307 char __user
*optval
, int __user
*optlen
);
308 int udp_lib_setsockopt(struct sock
*sk
, int level
, int optname
,
309 sockptr_t optval
, unsigned int optlen
,
310 int (*push_pending_frames
)(struct sock
*));
311 struct sock
*udp4_lib_lookup(struct net
*net
, __be32 saddr
, __be16 sport
,
312 __be32 daddr
, __be16 dport
, int dif
);
313 struct sock
*__udp4_lib_lookup(struct net
*net
, __be32 saddr
, __be16 sport
,
314 __be32 daddr
, __be16 dport
, int dif
, int sdif
,
315 struct udp_table
*tbl
, struct sk_buff
*skb
);
316 struct sock
*udp4_lib_lookup_skb(const struct sk_buff
*skb
,
317 __be16 sport
, __be16 dport
);
318 struct sock
*udp6_lib_lookup(struct net
*net
,
319 const struct in6_addr
*saddr
, __be16 sport
,
320 const struct in6_addr
*daddr
, __be16 dport
,
322 struct sock
*__udp6_lib_lookup(struct net
*net
,
323 const struct in6_addr
*saddr
, __be16 sport
,
324 const struct in6_addr
*daddr
, __be16 dport
,
325 int dif
, int sdif
, struct udp_table
*tbl
,
326 struct sk_buff
*skb
);
327 struct sock
*udp6_lib_lookup_skb(const struct sk_buff
*skb
,
328 __be16 sport
, __be16 dport
);
330 /* UDP uses skb->dev_scratch to cache as much information as possible and avoid
331 * possibly multiple cache miss on dequeue()
333 struct udp_dev_scratch
{
334 /* skb->truesize and the stateless bit are embedded in a single field;
335 * do not use a bitfield since the compiler emits better/smaller code
340 #if BITS_PER_LONG == 64
341 /* len and the bit needed to compute skb_csum_unnecessary
342 * will be on cold cache lines at recvmsg time.
343 * skb->len can be stored on 16 bits since the udp header has been
344 * already validated and pulled.
348 bool csum_unnecessary
;
352 static inline struct udp_dev_scratch
*udp_skb_scratch(struct sk_buff
*skb
)
354 return (struct udp_dev_scratch
*)&skb
->dev_scratch
;
357 #if BITS_PER_LONG == 64
358 static inline unsigned int udp_skb_len(struct sk_buff
*skb
)
360 return udp_skb_scratch(skb
)->len
;
363 static inline bool udp_skb_csum_unnecessary(struct sk_buff
*skb
)
365 return udp_skb_scratch(skb
)->csum_unnecessary
;
368 static inline bool udp_skb_is_linear(struct sk_buff
*skb
)
370 return udp_skb_scratch(skb
)->is_linear
;
374 static inline unsigned int udp_skb_len(struct sk_buff
*skb
)
379 static inline bool udp_skb_csum_unnecessary(struct sk_buff
*skb
)
381 return skb_csum_unnecessary(skb
);
384 static inline bool udp_skb_is_linear(struct sk_buff
*skb
)
386 return !skb_is_nonlinear(skb
);
390 static inline int copy_linear_skb(struct sk_buff
*skb
, int len
, int off
,
395 n
= copy_to_iter(skb
->data
+ off
, len
, to
);
399 iov_iter_revert(to
, n
);
404 * SNMP statistics for UDP and UDP-Lite
406 #define UDP_INC_STATS(net, field, is_udplite) do { \
407 if (is_udplite) SNMP_INC_STATS((net)->mib.udplite_statistics, field); \
408 else SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0)
409 #define __UDP_INC_STATS(net, field, is_udplite) do { \
410 if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_statistics, field); \
411 else __SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0)
413 #define __UDP6_INC_STATS(net, field, is_udplite) do { \
414 if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);\
415 else __SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \
417 #define UDP6_INC_STATS(net, field, __lite) do { \
418 if (__lite) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field); \
419 else SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \
422 #if IS_ENABLED(CONFIG_IPV6)
423 #define __UDPX_MIB(sk, ipv4) \
425 ipv4 ? (IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_statistics : \
426 sock_net(sk)->mib.udp_statistics) : \
427 (IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_stats_in6 : \
428 sock_net(sk)->mib.udp_stats_in6); \
431 #define __UDPX_MIB(sk, ipv4) \
433 IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_statistics : \
434 sock_net(sk)->mib.udp_statistics; \
438 #define __UDPX_INC_STATS(sk, field) \
439 __SNMP_INC_STATS(__UDPX_MIB(sk, (sk)->sk_family == AF_INET), field)
441 #ifdef CONFIG_PROC_FS
442 struct udp_seq_afinfo
{
444 struct udp_table
*udp_table
;
447 struct udp_iter_state
{
448 struct seq_net_private p
;
450 struct udp_seq_afinfo
*bpf_seq_afinfo
;
453 void *udp_seq_start(struct seq_file
*seq
, loff_t
*pos
);
454 void *udp_seq_next(struct seq_file
*seq
, void *v
, loff_t
*pos
);
455 void udp_seq_stop(struct seq_file
*seq
, void *v
);
457 extern const struct seq_operations udp_seq_ops
;
458 extern const struct seq_operations udp6_seq_ops
;
460 int udp4_proc_init(void);
461 void udp4_proc_exit(void);
462 #endif /* CONFIG_PROC_FS */
464 int udpv4_offload_init(void);
468 DECLARE_STATIC_KEY_FALSE(udp_encap_needed_key
);
469 void udp_encap_enable(void);
470 #if IS_ENABLED(CONFIG_IPV6)
471 DECLARE_STATIC_KEY_FALSE(udpv6_encap_needed_key
);
472 void udpv6_encap_enable(void);
475 static inline struct sk_buff
*udp_rcv_segment(struct sock
*sk
,
476 struct sk_buff
*skb
, bool ipv4
)
478 netdev_features_t features
= NETIF_F_SG
;
479 struct sk_buff
*segs
;
481 /* Avoid csum recalculation by skb_segment unless userspace explicitly
482 * asks for the final checksum values
484 if (!inet_get_convert_csum(sk
))
485 features
|= NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
;
487 /* UDP segmentation expects packets of type CHECKSUM_PARTIAL or
488 * CHECKSUM_NONE in __udp_gso_segment. UDP GRO indeed builds partial
489 * packets in udp_gro_complete_segment. As does UDP GSO, verified by
490 * udp_send_skb. But when those packets are looped in dev_loopback_xmit
491 * their ip_summed is set to CHECKSUM_UNNECESSARY. Reset in this
492 * specific case, where PARTIAL is both correct and required.
494 if (skb
->pkt_type
== PACKET_LOOPBACK
)
495 skb
->ip_summed
= CHECKSUM_PARTIAL
;
497 /* the GSO CB lays after the UDP one, no need to save and restore any
500 segs
= __skb_gso_segment(skb
, features
, false);
501 if (IS_ERR_OR_NULL(segs
)) {
502 int segs_nr
= skb_shinfo(skb
)->gso_segs
;
504 atomic_add(segs_nr
, &sk
->sk_drops
);
505 SNMP_ADD_STATS(__UDPX_MIB(sk
, ipv4
), UDP_MIB_INERRORS
, segs_nr
);
514 #ifdef CONFIG_BPF_STREAM_PARSER
516 struct proto
*udp_bpf_get_proto(struct sock
*sk
, struct sk_psock
*psock
);
517 #endif /* BPF_STREAM_PARSER */