]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - net/ipv4/udp.c
seq_file: introduce seq_setwidth() and seq_pad()
[mirror_ubuntu-zesty-kernel.git] / net / ipv4 / udp.c
CommitLineData
1da177e4
LT
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The User Datagram Protocol (UDP).
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
113aa838 11 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
1da177e4
LT
12 * Hirokazu Takahashi, <taka@valinux.co.jp>
13 *
14 * Fixes:
15 * Alan Cox : verify_area() calls
16 * Alan Cox : stopped close while in use off icmp
17 * messages. Not a fix but a botch that
18 * for udp at least is 'valid'.
19 * Alan Cox : Fixed icmp handling properly
20 * Alan Cox : Correct error for oversized datagrams
e905a9ed
YH
21 * Alan Cox : Tidied select() semantics.
22 * Alan Cox : udp_err() fixed properly, also now
1da177e4
LT
23 * select and read wake correctly on errors
24 * Alan Cox : udp_send verify_area moved to avoid mem leak
25 * Alan Cox : UDP can count its memory
26 * Alan Cox : send to an unknown connection causes
27 * an ECONNREFUSED off the icmp, but
28 * does NOT close.
29 * Alan Cox : Switched to new sk_buff handlers. No more backlog!
30 * Alan Cox : Using generic datagram code. Even smaller and the PEEK
31 * bug no longer crashes it.
32 * Fred Van Kempen : Net2e support for sk->broadcast.
33 * Alan Cox : Uses skb_free_datagram
34 * Alan Cox : Added get/set sockopt support.
35 * Alan Cox : Broadcasting without option set returns EACCES.
36 * Alan Cox : No wakeup calls. Instead we now use the callbacks.
37 * Alan Cox : Use ip_tos and ip_ttl
38 * Alan Cox : SNMP Mibs
39 * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
40 * Matt Dillon : UDP length checks.
41 * Alan Cox : Smarter af_inet used properly.
42 * Alan Cox : Use new kernel side addressing.
43 * Alan Cox : Incorrect return on truncated datagram receive.
44 * Arnt Gulbrandsen : New udp_send and stuff
45 * Alan Cox : Cache last socket
46 * Alan Cox : Route cache
47 * Jon Peatfield : Minor efficiency fix to sendto().
48 * Mike Shaver : RFC1122 checks.
49 * Alan Cox : Nonblocking error fix.
50 * Willy Konynenberg : Transparent proxying support.
51 * Mike McLagan : Routing by source
52 * David S. Miller : New socket lookup architecture.
53 * Last socket cache retained as it
54 * does have a high hit rate.
55 * Olaf Kirch : Don't linearise iovec on sendmsg.
56 * Andi Kleen : Some cleanups, cache destination entry
e905a9ed 57 * for connect.
1da177e4
LT
58 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
59 * Melvin Smith : Check msg_name not msg_namelen in sendto(),
60 * return ENOTCONN for unconnected sockets (POSIX)
61 * Janos Farkas : don't deliver multi/broadcasts to a different
62 * bound-to-device socket
63 * Hirokazu Takahashi : HW checksumming for outgoing UDP
64 * datagrams.
65 * Hirokazu Takahashi : sendfile() on UDP works now.
66 * Arnaldo C. Melo : convert /proc/net/udp to seq_file
67 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
68 * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
69 * a single port at the same time.
70 * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
342f0234 71 * James Chapman : Add L2TP encapsulation type.
1da177e4
LT
72 *
73 *
74 * This program is free software; you can redistribute it and/or
75 * modify it under the terms of the GNU General Public License
76 * as published by the Free Software Foundation; either version
77 * 2 of the License, or (at your option) any later version.
78 */
e905a9ed 79
afd46503
JP
80#define pr_fmt(fmt) "UDP: " fmt
81
1da177e4
LT
82#include <asm/uaccess.h>
83#include <asm/ioctls.h>
95766fff 84#include <linux/bootmem.h>
8203efb3
ED
85#include <linux/highmem.h>
86#include <linux/swap.h>
1da177e4
LT
87#include <linux/types.h>
88#include <linux/fcntl.h>
89#include <linux/module.h>
90#include <linux/socket.h>
91#include <linux/sockios.h>
14c85021 92#include <linux/igmp.h>
1da177e4
LT
93#include <linux/in.h>
94#include <linux/errno.h>
95#include <linux/timer.h>
96#include <linux/mm.h>
1da177e4 97#include <linux/inet.h>
1da177e4 98#include <linux/netdevice.h>
5a0e3ad6 99#include <linux/slab.h>
c752f073 100#include <net/tcp_states.h>
1da177e4
LT
101#include <linux/skbuff.h>
102#include <linux/proc_fs.h>
103#include <linux/seq_file.h>
457c4cbc 104#include <net/net_namespace.h>
1da177e4 105#include <net/icmp.h>
421b3885 106#include <net/inet_hashtables.h>
1da177e4 107#include <net/route.h>
1da177e4
LT
108#include <net/checksum.h>
109#include <net/xfrm.h>
296f7ea7 110#include <trace/events/udp.h>
447167bf 111#include <linux/static_key.h>
22911fc5 112#include <trace/events/skb.h>
076bb0c8 113#include <net/busy_poll.h>
ba4e58ec 114#include "udp_impl.h"
1da177e4 115
f86dcc5a 116struct udp_table udp_table __read_mostly;
645ca708 117EXPORT_SYMBOL(udp_table);
1da177e4 118
8d987e5c 119long sysctl_udp_mem[3] __read_mostly;
95766fff 120EXPORT_SYMBOL(sysctl_udp_mem);
c482c568
ED
121
122int sysctl_udp_rmem_min __read_mostly;
95766fff 123EXPORT_SYMBOL(sysctl_udp_rmem_min);
c482c568
ED
124
125int sysctl_udp_wmem_min __read_mostly;
95766fff
HA
126EXPORT_SYMBOL(sysctl_udp_wmem_min);
127
8d987e5c 128atomic_long_t udp_memory_allocated;
95766fff
HA
129EXPORT_SYMBOL(udp_memory_allocated);
130
f86dcc5a
ED
131#define MAX_UDP_PORTS 65536
132#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
98322f22 133
f24d43c0 134static int udp_lib_lport_inuse(struct net *net, __u16 num,
645ca708 135 const struct udp_hslot *hslot,
98322f22 136 unsigned long *bitmap,
f24d43c0
ED
137 struct sock *sk,
138 int (*saddr_comp)(const struct sock *sk1,
f86dcc5a
ED
139 const struct sock *sk2),
140 unsigned int log)
1da177e4 141{
f24d43c0 142 struct sock *sk2;
88ab1932 143 struct hlist_nulls_node *node;
ba418fa3 144 kuid_t uid = sock_i_uid(sk);
25030a7f 145
88ab1932 146 sk_nulls_for_each(sk2, node, &hslot->head)
9d4fb27d
JP
147 if (net_eq(sock_net(sk2), net) &&
148 sk2 != sk &&
d4cada4a 149 (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
9d4fb27d
JP
150 (!sk2->sk_reuse || !sk->sk_reuse) &&
151 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
152 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
ba418fa3
TH
153 (!sk2->sk_reuseport || !sk->sk_reuseport ||
154 !uid_eq(uid, sock_i_uid(sk2))) &&
98322f22
ED
155 (*saddr_comp)(sk, sk2)) {
156 if (bitmap)
d4cada4a
ED
157 __set_bit(udp_sk(sk2)->udp_port_hash >> log,
158 bitmap);
98322f22
ED
159 else
160 return 1;
161 }
25030a7f
GR
162 return 0;
163}
164
30fff923
ED
165/*
166 * Note: we still hold spinlock of primary hash chain, so no other writer
167 * can insert/delete a socket with local_port == num
168 */
169static int udp_lib_lport_inuse2(struct net *net, __u16 num,
170 struct udp_hslot *hslot2,
171 struct sock *sk,
172 int (*saddr_comp)(const struct sock *sk1,
173 const struct sock *sk2))
174{
175 struct sock *sk2;
176 struct hlist_nulls_node *node;
ba418fa3 177 kuid_t uid = sock_i_uid(sk);
30fff923
ED
178 int res = 0;
179
180 spin_lock(&hslot2->lock);
181 udp_portaddr_for_each_entry(sk2, node, &hslot2->head)
9d4fb27d
JP
182 if (net_eq(sock_net(sk2), net) &&
183 sk2 != sk &&
184 (udp_sk(sk2)->udp_port_hash == num) &&
185 (!sk2->sk_reuse || !sk->sk_reuse) &&
186 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
187 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
ba418fa3
TH
188 (!sk2->sk_reuseport || !sk->sk_reuseport ||
189 !uid_eq(uid, sock_i_uid(sk2))) &&
30fff923
ED
190 (*saddr_comp)(sk, sk2)) {
191 res = 1;
192 break;
193 }
194 spin_unlock(&hslot2->lock);
195 return res;
196}
197
25030a7f 198/**
6ba5a3c5 199 * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
25030a7f
GR
200 *
201 * @sk: socket struct in question
202 * @snum: port number to look up
df2bc459 203 * @saddr_comp: AF-dependent comparison of bound local IP addresses
25985edc 204 * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
30fff923 205 * with NULL address
25030a7f 206 */
6ba5a3c5 207int udp_lib_get_port(struct sock *sk, unsigned short snum,
df2bc459 208 int (*saddr_comp)(const struct sock *sk1,
30fff923
ED
209 const struct sock *sk2),
210 unsigned int hash2_nulladdr)
25030a7f 211{
512615b6 212 struct udp_hslot *hslot, *hslot2;
645ca708 213 struct udp_table *udptable = sk->sk_prot->h.udp_table;
25030a7f 214 int error = 1;
3b1e0a65 215 struct net *net = sock_net(sk);
1da177e4 216
32c1da70 217 if (!snum) {
9088c560 218 int low, high, remaining;
95c96174 219 unsigned int rand;
98322f22
ED
220 unsigned short first, last;
221 DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
32c1da70 222
0bbf87d8 223 inet_get_local_port_range(net, &low, &high);
a25de534 224 remaining = (high - low) + 1;
227b60f5 225
9088c560 226 rand = net_random();
98322f22
ED
227 first = (((u64)rand * remaining) >> 32) + low;
228 /*
229 * force rand to be an odd multiple of UDP_HTABLE_SIZE
230 */
f86dcc5a 231 rand = (rand | 1) * (udptable->mask + 1);
5781b235
ED
232 last = first + udptable->mask + 1;
233 do {
f86dcc5a 234 hslot = udp_hashslot(udptable, net, first);
98322f22 235 bitmap_zero(bitmap, PORTS_PER_CHAIN);
645ca708 236 spin_lock_bh(&hslot->lock);
98322f22 237 udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
f86dcc5a 238 saddr_comp, udptable->log);
98322f22
ED
239
240 snum = first;
241 /*
242 * Iterate on all possible values of snum for this hash.
243 * Using steps of an odd multiple of UDP_HTABLE_SIZE
244 * give us randomization and full range coverage.
245 */
9088c560 246 do {
98322f22 247 if (low <= snum && snum <= high &&
e3826f1e
AW
248 !test_bit(snum >> udptable->log, bitmap) &&
249 !inet_is_reserved_local_port(snum))
98322f22
ED
250 goto found;
251 snum += rand;
252 } while (snum != first);
253 spin_unlock_bh(&hslot->lock);
5781b235 254 } while (++first != last);
98322f22 255 goto fail;
645ca708 256 } else {
f86dcc5a 257 hslot = udp_hashslot(udptable, net, snum);
645ca708 258 spin_lock_bh(&hslot->lock);
30fff923
ED
259 if (hslot->count > 10) {
260 int exist;
261 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
262
263 slot2 &= udptable->mask;
264 hash2_nulladdr &= udptable->mask;
265
266 hslot2 = udp_hashslot2(udptable, slot2);
267 if (hslot->count < hslot2->count)
268 goto scan_primary_hash;
269
270 exist = udp_lib_lport_inuse2(net, snum, hslot2,
271 sk, saddr_comp);
272 if (!exist && (hash2_nulladdr != slot2)) {
273 hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
274 exist = udp_lib_lport_inuse2(net, snum, hslot2,
275 sk, saddr_comp);
276 }
277 if (exist)
278 goto fail_unlock;
279 else
280 goto found;
281 }
282scan_primary_hash:
f86dcc5a
ED
283 if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk,
284 saddr_comp, 0))
645ca708
ED
285 goto fail_unlock;
286 }
98322f22 287found:
c720c7e8 288 inet_sk(sk)->inet_num = snum;
d4cada4a
ED
289 udp_sk(sk)->udp_port_hash = snum;
290 udp_sk(sk)->udp_portaddr_hash ^= snum;
1da177e4 291 if (sk_unhashed(sk)) {
88ab1932 292 sk_nulls_add_node_rcu(sk, &hslot->head);
fdcc8aa9 293 hslot->count++;
c29a0bc4 294 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
512615b6
ED
295
296 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
297 spin_lock(&hslot2->lock);
298 hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
299 &hslot2->head);
300 hslot2->count++;
301 spin_unlock(&hslot2->lock);
1da177e4 302 }
25030a7f 303 error = 0;
645ca708
ED
304fail_unlock:
305 spin_unlock_bh(&hslot->lock);
1da177e4 306fail:
25030a7f
GR
307 return error;
308}
c482c568 309EXPORT_SYMBOL(udp_lib_get_port);
25030a7f 310
499923c7 311static int ipv4_rcv_saddr_equal(const struct sock *sk1, const struct sock *sk2)
db8dac20
DM
312{
313 struct inet_sock *inet1 = inet_sk(sk1), *inet2 = inet_sk(sk2);
314
c482c568 315 return (!ipv6_only_sock(sk2) &&
c720c7e8
ED
316 (!inet1->inet_rcv_saddr || !inet2->inet_rcv_saddr ||
317 inet1->inet_rcv_saddr == inet2->inet_rcv_saddr));
db8dac20
DM
318}
319
d4cada4a
ED
320static unsigned int udp4_portaddr_hash(struct net *net, __be32 saddr,
321 unsigned int port)
322{
0eae88f3 323 return jhash_1word((__force u32)saddr, net_hash_mix(net)) ^ port;
d4cada4a
ED
324}
325
6ba5a3c5 326int udp_v4_get_port(struct sock *sk, unsigned short snum)
db8dac20 327{
30fff923 328 unsigned int hash2_nulladdr =
0eae88f3 329 udp4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
30fff923
ED
330 unsigned int hash2_partial =
331 udp4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
332
d4cada4a 333 /* precompute partial secondary hash */
30fff923
ED
334 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
335 return udp_lib_get_port(sk, snum, ipv4_rcv_saddr_equal, hash2_nulladdr);
db8dac20
DM
336}
337
645ca708
ED
338static inline int compute_score(struct sock *sk, struct net *net, __be32 saddr,
339 unsigned short hnum,
340 __be16 sport, __be32 daddr, __be16 dport, int dif)
341{
342 int score = -1;
343
d4cada4a 344 if (net_eq(sock_net(sk), net) && udp_sk(sk)->udp_port_hash == hnum &&
645ca708
ED
345 !ipv6_only_sock(sk)) {
346 struct inet_sock *inet = inet_sk(sk);
347
ba418fa3 348 score = (sk->sk_family == PF_INET ? 2 : 1);
c720c7e8
ED
349 if (inet->inet_rcv_saddr) {
350 if (inet->inet_rcv_saddr != daddr)
645ca708 351 return -1;
ba418fa3 352 score += 4;
645ca708 353 }
c720c7e8
ED
354 if (inet->inet_daddr) {
355 if (inet->inet_daddr != saddr)
645ca708 356 return -1;
ba418fa3 357 score += 4;
645ca708 358 }
c720c7e8
ED
359 if (inet->inet_dport) {
360 if (inet->inet_dport != sport)
645ca708 361 return -1;
ba418fa3 362 score += 4;
645ca708
ED
363 }
364 if (sk->sk_bound_dev_if) {
365 if (sk->sk_bound_dev_if != dif)
366 return -1;
ba418fa3 367 score += 4;
645ca708
ED
368 }
369 }
370 return score;
371}
372
5051ebd2
ED
373/*
374 * In this second variant, we check (daddr, dport) matches (inet_rcv_sadd, inet_num)
375 */
5051ebd2
ED
376static inline int compute_score2(struct sock *sk, struct net *net,
377 __be32 saddr, __be16 sport,
378 __be32 daddr, unsigned int hnum, int dif)
379{
380 int score = -1;
381
382 if (net_eq(sock_net(sk), net) && !ipv6_only_sock(sk)) {
383 struct inet_sock *inet = inet_sk(sk);
384
385 if (inet->inet_rcv_saddr != daddr)
386 return -1;
387 if (inet->inet_num != hnum)
388 return -1;
389
ba418fa3 390 score = (sk->sk_family == PF_INET ? 2 : 1);
5051ebd2
ED
391 if (inet->inet_daddr) {
392 if (inet->inet_daddr != saddr)
393 return -1;
ba418fa3 394 score += 4;
5051ebd2
ED
395 }
396 if (inet->inet_dport) {
397 if (inet->inet_dport != sport)
398 return -1;
ba418fa3 399 score += 4;
5051ebd2
ED
400 }
401 if (sk->sk_bound_dev_if) {
402 if (sk->sk_bound_dev_if != dif)
403 return -1;
ba418fa3 404 score += 4;
5051ebd2
ED
405 }
406 }
407 return score;
408}
409
65cd8033
HFS
410static unsigned int udp_ehashfn(struct net *net, const __be32 laddr,
411 const __u16 lport, const __be32 faddr,
412 const __be16 fport)
413{
1bbdceef
HFS
414 static u32 udp_ehash_secret __read_mostly;
415
416 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
417
65cd8033 418 return __inet_ehashfn(laddr, lport, faddr, fport,
1bbdceef 419 udp_ehash_secret + net_hash_mix(net));
65cd8033
HFS
420}
421
5051ebd2
ED
422
423/* called with read_rcu_lock() */
424static struct sock *udp4_lib_lookup2(struct net *net,
425 __be32 saddr, __be16 sport,
426 __be32 daddr, unsigned int hnum, int dif,
427 struct udp_hslot *hslot2, unsigned int slot2)
428{
429 struct sock *sk, *result;
430 struct hlist_nulls_node *node;
ba418fa3
TH
431 int score, badness, matches = 0, reuseport = 0;
432 u32 hash = 0;
5051ebd2
ED
433
434begin:
435 result = NULL;
ba418fa3 436 badness = 0;
5051ebd2
ED
437 udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
438 score = compute_score2(sk, net, saddr, sport,
439 daddr, hnum, dif);
440 if (score > badness) {
441 result = sk;
442 badness = score;
ba418fa3
TH
443 reuseport = sk->sk_reuseport;
444 if (reuseport) {
65cd8033
HFS
445 hash = udp_ehashfn(net, daddr, hnum,
446 saddr, sport);
ba418fa3
TH
447 matches = 1;
448 }
449 } else if (score == badness && reuseport) {
450 matches++;
451 if (((u64)hash * matches) >> 32 == 0)
452 result = sk;
453 hash = next_pseudo_random32(hash);
5051ebd2
ED
454 }
455 }
456 /*
457 * if the nulls value we got at the end of this lookup is
458 * not the expected one, we must restart lookup.
459 * We probably met an item that was moved to another chain.
460 */
461 if (get_nulls_value(node) != slot2)
462 goto begin;
5051ebd2 463 if (result) {
c31504dc 464 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
5051ebd2
ED
465 result = NULL;
466 else if (unlikely(compute_score2(result, net, saddr, sport,
467 daddr, hnum, dif) < badness)) {
468 sock_put(result);
469 goto begin;
470 }
471 }
472 return result;
473}
474
db8dac20
DM
475/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
476 * harder than this. -DaveM
477 */
fce82338 478struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
db8dac20 479 __be16 sport, __be32 daddr, __be16 dport,
645ca708 480 int dif, struct udp_table *udptable)
db8dac20 481{
271b72c7 482 struct sock *sk, *result;
88ab1932 483 struct hlist_nulls_node *node;
db8dac20 484 unsigned short hnum = ntohs(dport);
5051ebd2
ED
485 unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
486 struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
ba418fa3
TH
487 int score, badness, matches = 0, reuseport = 0;
488 u32 hash = 0;
645ca708 489
271b72c7 490 rcu_read_lock();
5051ebd2
ED
491 if (hslot->count > 10) {
492 hash2 = udp4_portaddr_hash(net, daddr, hnum);
493 slot2 = hash2 & udptable->mask;
494 hslot2 = &udptable->hash2[slot2];
495 if (hslot->count < hslot2->count)
496 goto begin;
497
498 result = udp4_lib_lookup2(net, saddr, sport,
499 daddr, hnum, dif,
500 hslot2, slot2);
501 if (!result) {
0eae88f3 502 hash2 = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
5051ebd2
ED
503 slot2 = hash2 & udptable->mask;
504 hslot2 = &udptable->hash2[slot2];
505 if (hslot->count < hslot2->count)
506 goto begin;
507
1223c67c 508 result = udp4_lib_lookup2(net, saddr, sport,
0eae88f3 509 htonl(INADDR_ANY), hnum, dif,
5051ebd2
ED
510 hslot2, slot2);
511 }
512 rcu_read_unlock();
513 return result;
514 }
271b72c7
ED
515begin:
516 result = NULL;
ba418fa3 517 badness = 0;
88ab1932 518 sk_nulls_for_each_rcu(sk, node, &hslot->head) {
645ca708
ED
519 score = compute_score(sk, net, saddr, hnum, sport,
520 daddr, dport, dif);
521 if (score > badness) {
522 result = sk;
523 badness = score;
ba418fa3
TH
524 reuseport = sk->sk_reuseport;
525 if (reuseport) {
65cd8033
HFS
526 hash = udp_ehashfn(net, daddr, hnum,
527 saddr, sport);
ba418fa3
TH
528 matches = 1;
529 }
530 } else if (score == badness && reuseport) {
531 matches++;
532 if (((u64)hash * matches) >> 32 == 0)
533 result = sk;
534 hash = next_pseudo_random32(hash);
db8dac20
DM
535 }
536 }
88ab1932
ED
537 /*
538 * if the nulls value we got at the end of this lookup is
539 * not the expected one, we must restart lookup.
540 * We probably met an item that was moved to another chain.
541 */
5051ebd2 542 if (get_nulls_value(node) != slot)
88ab1932
ED
543 goto begin;
544
271b72c7 545 if (result) {
c31504dc 546 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
271b72c7
ED
547 result = NULL;
548 else if (unlikely(compute_score(result, net, saddr, hnum, sport,
549 daddr, dport, dif) < badness)) {
550 sock_put(result);
551 goto begin;
552 }
553 }
554 rcu_read_unlock();
db8dac20
DM
555 return result;
556}
fce82338 557EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
db8dac20 558
607c4aaf
KK
559static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
560 __be16 sport, __be16 dport,
645ca708 561 struct udp_table *udptable)
607c4aaf 562{
23542618 563 struct sock *sk;
607c4aaf
KK
564 const struct iphdr *iph = ip_hdr(skb);
565
23542618
KK
566 if (unlikely(sk = skb_steal_sock(skb)))
567 return sk;
568 else
adf30907 569 return __udp4_lib_lookup(dev_net(skb_dst(skb)->dev), iph->saddr, sport,
23542618
KK
570 iph->daddr, dport, inet_iif(skb),
571 udptable);
607c4aaf
KK
572}
573
bcd41303
KK
574struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
575 __be32 daddr, __be16 dport, int dif)
576{
645ca708 577 return __udp4_lib_lookup(net, saddr, sport, daddr, dport, dif, &udp_table);
bcd41303
KK
578}
579EXPORT_SYMBOL_GPL(udp4_lib_lookup);
580
421b3885
SB
581static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
582 __be16 loc_port, __be32 loc_addr,
583 __be16 rmt_port, __be32 rmt_addr,
584 int dif, unsigned short hnum)
585{
586 struct inet_sock *inet = inet_sk(sk);
587
588 if (!net_eq(sock_net(sk), net) ||
589 udp_sk(sk)->udp_port_hash != hnum ||
590 (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
591 (inet->inet_dport != rmt_port && inet->inet_dport) ||
592 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
593 ipv6_only_sock(sk) ||
594 (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif))
595 return false;
596 if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif))
597 return false;
598 return true;
599}
600
920a4611 601static inline struct sock *udp_v4_mcast_next(struct net *net, struct sock *sk,
db8dac20
DM
602 __be16 loc_port, __be32 loc_addr,
603 __be16 rmt_port, __be32 rmt_addr,
604 int dif)
605{
88ab1932 606 struct hlist_nulls_node *node;
db8dac20
DM
607 struct sock *s = sk;
608 unsigned short hnum = ntohs(loc_port);
609
88ab1932 610 sk_nulls_for_each_from(s, node) {
421b3885
SB
611 if (__udp_is_mcast_sock(net, s,
612 loc_port, loc_addr,
613 rmt_port, rmt_addr,
614 dif, hnum))
615 goto found;
db8dac20
DM
616 }
617 s = NULL;
618found:
619 return s;
620}
621
622/*
623 * This routine is called by the ICMP module when it gets some
624 * sort of error condition. If err < 0 then the socket should
625 * be closed and the error returned to the user. If err > 0
626 * it's just the icmp type << 8 | icmp code.
627 * Header points to the ip header of the error packet. We move
628 * on past this. Then (as it used to claim before adjustment)
629 * header points to the first 8 bytes of the udp header. We need
630 * to find the appropriate port.
631 */
632
645ca708 633void __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
db8dac20
DM
634{
635 struct inet_sock *inet;
b71d1d42 636 const struct iphdr *iph = (const struct iphdr *)skb->data;
c482c568 637 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
db8dac20
DM
638 const int type = icmp_hdr(skb)->type;
639 const int code = icmp_hdr(skb)->code;
640 struct sock *sk;
641 int harderr;
642 int err;
fd54d716 643 struct net *net = dev_net(skb->dev);
db8dac20 644
fd54d716 645 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
db8dac20
DM
646 iph->saddr, uh->source, skb->dev->ifindex, udptable);
647 if (sk == NULL) {
dcfc23ca 648 ICMP_INC_STATS_BH(net, ICMP_MIB_INERRORS);
db8dac20
DM
649 return; /* No socket for error */
650 }
651
652 err = 0;
653 harderr = 0;
654 inet = inet_sk(sk);
655
656 switch (type) {
657 default:
658 case ICMP_TIME_EXCEEDED:
659 err = EHOSTUNREACH;
660 break;
661 case ICMP_SOURCE_QUENCH:
662 goto out;
663 case ICMP_PARAMETERPROB:
664 err = EPROTO;
665 harderr = 1;
666 break;
667 case ICMP_DEST_UNREACH:
668 if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
36393395 669 ipv4_sk_update_pmtu(skb, sk, info);
db8dac20
DM
670 if (inet->pmtudisc != IP_PMTUDISC_DONT) {
671 err = EMSGSIZE;
672 harderr = 1;
673 break;
674 }
675 goto out;
676 }
677 err = EHOSTUNREACH;
678 if (code <= NR_ICMP_UNREACH) {
679 harderr = icmp_err_convert[code].fatal;
680 err = icmp_err_convert[code].errno;
681 }
682 break;
55be7a9c
DM
683 case ICMP_REDIRECT:
684 ipv4_sk_redirect(skb, sk);
1a462d18 685 goto out;
db8dac20
DM
686 }
687
688 /*
689 * RFC1122: OK. Passes ICMP errors back to application, as per
690 * 4.1.3.3.
691 */
692 if (!inet->recverr) {
693 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
694 goto out;
b1faf566 695 } else
c482c568 696 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
b1faf566 697
db8dac20
DM
698 sk->sk_err = err;
699 sk->sk_error_report(sk);
700out:
701 sock_put(sk);
702}
703
704void udp_err(struct sk_buff *skb, u32 info)
705{
645ca708 706 __udp4_lib_err(skb, info, &udp_table);
db8dac20
DM
707}
708
709/*
710 * Throw away all pending data and cancel the corking. Socket is locked.
711 */
36d926b9 712void udp_flush_pending_frames(struct sock *sk)
db8dac20
DM
713{
714 struct udp_sock *up = udp_sk(sk);
715
716 if (up->pending) {
717 up->len = 0;
718 up->pending = 0;
719 ip_flush_pending_frames(sk);
720 }
721}
36d926b9 722EXPORT_SYMBOL(udp_flush_pending_frames);
db8dac20
DM
723
724/**
f6b9664f 725 * udp4_hwcsum - handle outgoing HW checksumming
db8dac20
DM
726 * @skb: sk_buff containing the filled-in UDP header
727 * (checksum field must be zeroed out)
f6b9664f
HX
728 * @src: source IP address
729 * @dst: destination IP address
db8dac20 730 */
c26bf4a5 731void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
db8dac20 732{
db8dac20 733 struct udphdr *uh = udp_hdr(skb);
f6b9664f
HX
734 struct sk_buff *frags = skb_shinfo(skb)->frag_list;
735 int offset = skb_transport_offset(skb);
736 int len = skb->len - offset;
737 int hlen = len;
db8dac20
DM
738 __wsum csum = 0;
739
f6b9664f 740 if (!frags) {
db8dac20
DM
741 /*
742 * Only one fragment on the socket.
743 */
744 skb->csum_start = skb_transport_header(skb) - skb->head;
745 skb->csum_offset = offsetof(struct udphdr, check);
f6b9664f
HX
746 uh->check = ~csum_tcpudp_magic(src, dst, len,
747 IPPROTO_UDP, 0);
db8dac20
DM
748 } else {
749 /*
750 * HW-checksum won't work as there are two or more
751 * fragments on the socket so that all csums of sk_buffs
752 * should be together
753 */
f6b9664f
HX
754 do {
755 csum = csum_add(csum, frags->csum);
756 hlen -= frags->len;
757 } while ((frags = frags->next));
db8dac20 758
f6b9664f 759 csum = skb_checksum(skb, offset, hlen, csum);
db8dac20
DM
760 skb->ip_summed = CHECKSUM_NONE;
761
db8dac20
DM
762 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
763 if (uh->check == 0)
764 uh->check = CSUM_MANGLED_0;
765 }
766}
c26bf4a5 767EXPORT_SYMBOL_GPL(udp4_hwcsum);
db8dac20 768
79ab0531 769static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4)
db8dac20 770{
f6b9664f 771 struct sock *sk = skb->sk;
db8dac20 772 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM
773 struct udphdr *uh;
774 int err = 0;
775 int is_udplite = IS_UDPLITE(sk);
f6b9664f
HX
776 int offset = skb_transport_offset(skb);
777 int len = skb->len - offset;
db8dac20
DM
778 __wsum csum = 0;
779
db8dac20
DM
780 /*
781 * Create a UDP header
782 */
783 uh = udp_hdr(skb);
f6b9664f 784 uh->source = inet->inet_sport;
79ab0531 785 uh->dest = fl4->fl4_dport;
f6b9664f 786 uh->len = htons(len);
db8dac20
DM
787 uh->check = 0;
788
789 if (is_udplite) /* UDP-Lite */
f6b9664f 790 csum = udplite_csum(skb);
db8dac20
DM
791
792 else if (sk->sk_no_check == UDP_CSUM_NOXMIT) { /* UDP csum disabled */
793
794 skb->ip_summed = CHECKSUM_NONE;
795 goto send;
796
797 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
798
79ab0531 799 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
db8dac20
DM
800 goto send;
801
f6b9664f
HX
802 } else
803 csum = udp_csum(skb);
db8dac20
DM
804
805 /* add protocol-dependent pseudo-header */
79ab0531 806 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
c482c568 807 sk->sk_protocol, csum);
db8dac20
DM
808 if (uh->check == 0)
809 uh->check = CSUM_MANGLED_0;
810
811send:
b5ec8eea 812 err = ip_send_skb(sock_net(sk), skb);
6ce9e7b5
ED
813 if (err) {
814 if (err == -ENOBUFS && !inet->recverr) {
815 UDP_INC_STATS_USER(sock_net(sk),
816 UDP_MIB_SNDBUFERRORS, is_udplite);
817 err = 0;
818 }
819 } else
820 UDP_INC_STATS_USER(sock_net(sk),
821 UDP_MIB_OUTDATAGRAMS, is_udplite);
f6b9664f
HX
822 return err;
823}
824
825/*
826 * Push out all pending data as one UDP datagram. Socket is locked.
827 */
8822b64a 828int udp_push_pending_frames(struct sock *sk)
f6b9664f
HX
829{
830 struct udp_sock *up = udp_sk(sk);
831 struct inet_sock *inet = inet_sk(sk);
b6f21b26 832 struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
f6b9664f
HX
833 struct sk_buff *skb;
834 int err = 0;
835
77968b78 836 skb = ip_finish_skb(sk, fl4);
f6b9664f
HX
837 if (!skb)
838 goto out;
839
79ab0531 840 err = udp_send_skb(skb, fl4);
f6b9664f 841
db8dac20
DM
842out:
843 up->len = 0;
844 up->pending = 0;
db8dac20
DM
845 return err;
846}
8822b64a 847EXPORT_SYMBOL(udp_push_pending_frames);
db8dac20
DM
848
849int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
850 size_t len)
851{
852 struct inet_sock *inet = inet_sk(sk);
853 struct udp_sock *up = udp_sk(sk);
e474995f 854 struct flowi4 fl4_stack;
b6f21b26 855 struct flowi4 *fl4;
db8dac20
DM
856 int ulen = len;
857 struct ipcm_cookie ipc;
858 struct rtable *rt = NULL;
859 int free = 0;
860 int connected = 0;
861 __be32 daddr, faddr, saddr;
862 __be16 dport;
863 u8 tos;
864 int err, is_udplite = IS_UDPLITE(sk);
865 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
866 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
903ab86d 867 struct sk_buff *skb;
f6d8bd05 868 struct ip_options_data opt_copy;
db8dac20
DM
869
870 if (len > 0xFFFF)
871 return -EMSGSIZE;
872
873 /*
874 * Check the flags.
875 */
876
c482c568 877 if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
db8dac20
DM
878 return -EOPNOTSUPP;
879
880 ipc.opt = NULL;
2244d07b 881 ipc.tx_flags = 0;
aa661581
FF
882 ipc.ttl = 0;
883 ipc.tos = -1;
db8dac20 884
903ab86d
HX
885 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
886
f5fca608 887 fl4 = &inet->cork.fl.u.ip4;
db8dac20
DM
888 if (up->pending) {
889 /*
890 * There are pending frames.
891 * The socket lock must be held while it's corked.
892 */
893 lock_sock(sk);
894 if (likely(up->pending)) {
895 if (unlikely(up->pending != AF_INET)) {
896 release_sock(sk);
897 return -EINVAL;
898 }
899 goto do_append_data;
900 }
901 release_sock(sk);
902 }
903 ulen += sizeof(struct udphdr);
904
905 /*
906 * Get and verify the address.
907 */
908 if (msg->msg_name) {
5e73ea1a 909 struct sockaddr_in *usin = (struct sockaddr_in *)msg->msg_name;
db8dac20
DM
910 if (msg->msg_namelen < sizeof(*usin))
911 return -EINVAL;
912 if (usin->sin_family != AF_INET) {
913 if (usin->sin_family != AF_UNSPEC)
914 return -EAFNOSUPPORT;
915 }
916
917 daddr = usin->sin_addr.s_addr;
918 dport = usin->sin_port;
919 if (dport == 0)
920 return -EINVAL;
921 } else {
922 if (sk->sk_state != TCP_ESTABLISHED)
923 return -EDESTADDRREQ;
c720c7e8
ED
924 daddr = inet->inet_daddr;
925 dport = inet->inet_dport;
db8dac20
DM
926 /* Open fast path for connected socket.
927 Route will not be used, if at least one option is set.
928 */
929 connected = 1;
930 }
c720c7e8 931 ipc.addr = inet->inet_saddr;
db8dac20
DM
932
933 ipc.oif = sk->sk_bound_dev_if;
bf84a010
DB
934
935 sock_tx_timestamp(sk, &ipc.tx_flags);
936
db8dac20 937 if (msg->msg_controllen) {
3b1e0a65 938 err = ip_cmsg_send(sock_net(sk), msg, &ipc);
db8dac20
DM
939 if (err)
940 return err;
941 if (ipc.opt)
942 free = 1;
943 connected = 0;
944 }
f6d8bd05
ED
945 if (!ipc.opt) {
946 struct ip_options_rcu *inet_opt;
947
948 rcu_read_lock();
949 inet_opt = rcu_dereference(inet->inet_opt);
950 if (inet_opt) {
951 memcpy(&opt_copy, inet_opt,
952 sizeof(*inet_opt) + inet_opt->opt.optlen);
953 ipc.opt = &opt_copy.opt;
954 }
955 rcu_read_unlock();
956 }
db8dac20
DM
957
958 saddr = ipc.addr;
959 ipc.addr = faddr = daddr;
960
f6d8bd05 961 if (ipc.opt && ipc.opt->opt.srr) {
db8dac20
DM
962 if (!daddr)
963 return -EINVAL;
f6d8bd05 964 faddr = ipc.opt->opt.faddr;
db8dac20
DM
965 connected = 0;
966 }
aa661581 967 tos = get_rttos(&ipc, inet);
db8dac20
DM
968 if (sock_flag(sk, SOCK_LOCALROUTE) ||
969 (msg->msg_flags & MSG_DONTROUTE) ||
f6d8bd05 970 (ipc.opt && ipc.opt->opt.is_strictroute)) {
db8dac20
DM
971 tos |= RTO_ONLINK;
972 connected = 0;
973 }
974
975 if (ipv4_is_multicast(daddr)) {
976 if (!ipc.oif)
977 ipc.oif = inet->mc_index;
978 if (!saddr)
979 saddr = inet->mc_addr;
980 connected = 0;
76e21053
EH
981 } else if (!ipc.oif)
982 ipc.oif = inet->uc_index;
db8dac20
DM
983
984 if (connected)
c482c568 985 rt = (struct rtable *)sk_dst_check(sk, 0);
db8dac20
DM
986
987 if (rt == NULL) {
84a3aa00
PE
988 struct net *net = sock_net(sk);
989
e474995f
DM
990 fl4 = &fl4_stack;
991 flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
c0951cbc
DM
992 RT_SCOPE_UNIVERSE, sk->sk_protocol,
993 inet_sk_flowi_flags(sk)|FLOWI_FLAG_CAN_SLEEP,
994 faddr, saddr, dport, inet->inet_sport);
995
e474995f
DM
996 security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
997 rt = ip_route_output_flow(net, fl4, sk);
b23dd4fe
DM
998 if (IS_ERR(rt)) {
999 err = PTR_ERR(rt);
06dc94b1 1000 rt = NULL;
db8dac20 1001 if (err == -ENETUNREACH)
7c73a6fa 1002 IP_INC_STATS_BH(net, IPSTATS_MIB_OUTNOROUTES);
db8dac20
DM
1003 goto out;
1004 }
1005
1006 err = -EACCES;
1007 if ((rt->rt_flags & RTCF_BROADCAST) &&
1008 !sock_flag(sk, SOCK_BROADCAST))
1009 goto out;
1010 if (connected)
d8d1f30b 1011 sk_dst_set(sk, dst_clone(&rt->dst));
db8dac20
DM
1012 }
1013
1014 if (msg->msg_flags&MSG_CONFIRM)
1015 goto do_confirm;
1016back_from_confirm:
1017
e474995f 1018 saddr = fl4->saddr;
db8dac20 1019 if (!ipc.addr)
e474995f 1020 daddr = ipc.addr = fl4->daddr;
db8dac20 1021
903ab86d
HX
1022 /* Lockless fast path for the non-corking case. */
1023 if (!corkreq) {
77968b78 1024 skb = ip_make_skb(sk, fl4, getfrag, msg->msg_iov, ulen,
903ab86d
HX
1025 sizeof(struct udphdr), &ipc, &rt,
1026 msg->msg_flags);
1027 err = PTR_ERR(skb);
50c3a487 1028 if (!IS_ERR_OR_NULL(skb))
79ab0531 1029 err = udp_send_skb(skb, fl4);
903ab86d
HX
1030 goto out;
1031 }
1032
db8dac20
DM
1033 lock_sock(sk);
1034 if (unlikely(up->pending)) {
1035 /* The socket is already corked while preparing it. */
1036 /* ... which is an evident application bug. --ANK */
1037 release_sock(sk);
1038
afd46503 1039 LIMIT_NETDEBUG(KERN_DEBUG pr_fmt("cork app bug 2\n"));
db8dac20
DM
1040 err = -EINVAL;
1041 goto out;
1042 }
1043 /*
1044 * Now cork the socket to pend data.
1045 */
b6f21b26
DM
1046 fl4 = &inet->cork.fl.u.ip4;
1047 fl4->daddr = daddr;
1048 fl4->saddr = saddr;
9cce96df
DM
1049 fl4->fl4_dport = dport;
1050 fl4->fl4_sport = inet->inet_sport;
db8dac20
DM
1051 up->pending = AF_INET;
1052
1053do_append_data:
1054 up->len += ulen;
f5fca608
DM
1055 err = ip_append_data(sk, fl4, getfrag, msg->msg_iov, ulen,
1056 sizeof(struct udphdr), &ipc, &rt,
1057 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
db8dac20
DM
1058 if (err)
1059 udp_flush_pending_frames(sk);
1060 else if (!corkreq)
1061 err = udp_push_pending_frames(sk);
1062 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1063 up->pending = 0;
1064 release_sock(sk);
1065
1066out:
1067 ip_rt_put(rt);
1068 if (free)
1069 kfree(ipc.opt);
1070 if (!err)
1071 return len;
1072 /*
1073 * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
1074 * ENOBUFS might not be good (it's not tunable per se), but otherwise
1075 * we don't have a good statistic (IpOutDiscards but it can be too many
1076 * things). We could add another new stat but at least for now that
1077 * seems like overkill.
1078 */
1079 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
629ca23c
PE
1080 UDP_INC_STATS_USER(sock_net(sk),
1081 UDP_MIB_SNDBUFERRORS, is_udplite);
db8dac20
DM
1082 }
1083 return err;
1084
1085do_confirm:
d8d1f30b 1086 dst_confirm(&rt->dst);
db8dac20
DM
1087 if (!(msg->msg_flags&MSG_PROBE) || len)
1088 goto back_from_confirm;
1089 err = 0;
1090 goto out;
1091}
c482c568 1092EXPORT_SYMBOL(udp_sendmsg);
db8dac20
DM
1093
1094int udp_sendpage(struct sock *sk, struct page *page, int offset,
1095 size_t size, int flags)
1096{
f5fca608 1097 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM
1098 struct udp_sock *up = udp_sk(sk);
1099 int ret;
1100
1101 if (!up->pending) {
1102 struct msghdr msg = { .msg_flags = flags|MSG_MORE };
1103
1104 /* Call udp_sendmsg to specify destination address which
1105 * sendpage interface can't pass.
1106 * This will succeed only when the socket is connected.
1107 */
1108 ret = udp_sendmsg(NULL, sk, &msg, 0);
1109 if (ret < 0)
1110 return ret;
1111 }
1112
1113 lock_sock(sk);
1114
1115 if (unlikely(!up->pending)) {
1116 release_sock(sk);
1117
afd46503 1118 LIMIT_NETDEBUG(KERN_DEBUG pr_fmt("udp cork app bug 3\n"));
db8dac20
DM
1119 return -EINVAL;
1120 }
1121
f5fca608
DM
1122 ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
1123 page, offset, size, flags);
db8dac20
DM
1124 if (ret == -EOPNOTSUPP) {
1125 release_sock(sk);
1126 return sock_no_sendpage(sk->sk_socket, page, offset,
1127 size, flags);
1128 }
1129 if (ret < 0) {
1130 udp_flush_pending_frames(sk);
1131 goto out;
1132 }
1133
1134 up->len += size;
1135 if (!(up->corkflag || (flags&MSG_MORE)))
1136 ret = udp_push_pending_frames(sk);
1137 if (!ret)
1138 ret = size;
1139out:
1140 release_sock(sk);
1141 return ret;
1142}
1143
85584672
ED
1144
1145/**
1146 * first_packet_length - return length of first packet in receive queue
1147 * @sk: socket
1148 *
1149 * Drops all bad checksum frames, until a valid one is found.
1150 * Returns the length of found skb, or 0 if none is found.
1151 */
1152static unsigned int first_packet_length(struct sock *sk)
1153{
1154 struct sk_buff_head list_kill, *rcvq = &sk->sk_receive_queue;
1155 struct sk_buff *skb;
1156 unsigned int res;
1157
1158 __skb_queue_head_init(&list_kill);
1159
1160 spin_lock_bh(&rcvq->lock);
1161 while ((skb = skb_peek(rcvq)) != NULL &&
1162 udp_lib_checksum_complete(skb)) {
6a5dc9e5
ED
1163 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS,
1164 IS_UDPLITE(sk));
85584672
ED
1165 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
1166 IS_UDPLITE(sk));
8edf19c2 1167 atomic_inc(&sk->sk_drops);
85584672
ED
1168 __skb_unlink(skb, rcvq);
1169 __skb_queue_tail(&list_kill, skb);
1170 }
1171 res = skb ? skb->len : 0;
1172 spin_unlock_bh(&rcvq->lock);
1173
1174 if (!skb_queue_empty(&list_kill)) {
8a74ad60
ED
1175 bool slow = lock_sock_fast(sk);
1176
85584672
ED
1177 __skb_queue_purge(&list_kill);
1178 sk_mem_reclaim_partial(sk);
8a74ad60 1179 unlock_sock_fast(sk, slow);
85584672
ED
1180 }
1181 return res;
1182}
1183
1da177e4
LT
1184/*
1185 * IOCTL requests applicable to the UDP protocol
1186 */
e905a9ed 1187
1da177e4
LT
1188int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
1189{
6516c655
SH
1190 switch (cmd) {
1191 case SIOCOUTQ:
1da177e4 1192 {
31e6d363
ED
1193 int amount = sk_wmem_alloc_get(sk);
1194
6516c655
SH
1195 return put_user(amount, (int __user *)arg);
1196 }
1da177e4 1197
6516c655
SH
1198 case SIOCINQ:
1199 {
85584672 1200 unsigned int amount = first_packet_length(sk);
6516c655 1201
85584672 1202 if (amount)
6516c655
SH
1203 /*
1204 * We will only return the amount
1205 * of this packet since that is all
1206 * that will be read.
1207 */
85584672
ED
1208 amount -= sizeof(struct udphdr);
1209
6516c655
SH
1210 return put_user(amount, (int __user *)arg);
1211 }
1da177e4 1212
6516c655
SH
1213 default:
1214 return -ENOIOCTLCMD;
1da177e4 1215 }
6516c655
SH
1216
1217 return 0;
1da177e4 1218}
c482c568 1219EXPORT_SYMBOL(udp_ioctl);
1da177e4 1220
db8dac20
DM
1221/*
1222 * This should be easy, if there is something there we
1223 * return it, otherwise we block.
1224 */
1225
1226int udp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1227 size_t len, int noblock, int flags, int *addr_len)
1228{
1229 struct inet_sock *inet = inet_sk(sk);
1230 struct sockaddr_in *sin = (struct sockaddr_in *)msg->msg_name;
1231 struct sk_buff *skb;
59c2cdae 1232 unsigned int ulen, copied;
3f518bf7 1233 int peeked, off = 0;
db8dac20
DM
1234 int err;
1235 int is_udplite = IS_UDPLITE(sk);
8a74ad60 1236 bool slow;
db8dac20
DM
1237
1238 /*
1239 * Check any passed addresses
1240 */
1241 if (addr_len)
c482c568 1242 *addr_len = sizeof(*sin);
db8dac20
DM
1243
1244 if (flags & MSG_ERRQUEUE)
1245 return ip_recv_error(sk, msg, len);
1246
1247try_again:
1248 skb = __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),
3f518bf7 1249 &peeked, &off, &err);
db8dac20
DM
1250 if (!skb)
1251 goto out;
1252
1253 ulen = skb->len - sizeof(struct udphdr);
59c2cdae
DM
1254 copied = len;
1255 if (copied > ulen)
1256 copied = ulen;
1257 else if (copied < ulen)
db8dac20
DM
1258 msg->msg_flags |= MSG_TRUNC;
1259
1260 /*
1261 * If checksum is needed at all, try to do it while copying the
1262 * data. If the data is truncated, or if we only want a partial
1263 * coverage checksum (UDP-Lite), do it before the copy.
1264 */
1265
59c2cdae 1266 if (copied < ulen || UDP_SKB_CB(skb)->partial_cov) {
db8dac20
DM
1267 if (udp_lib_checksum_complete(skb))
1268 goto csum_copy_err;
1269 }
1270
1271 if (skb_csum_unnecessary(skb))
1272 err = skb_copy_datagram_iovec(skb, sizeof(struct udphdr),
59c2cdae 1273 msg->msg_iov, copied);
db8dac20 1274 else {
c482c568
ED
1275 err = skb_copy_and_csum_datagram_iovec(skb,
1276 sizeof(struct udphdr),
1277 msg->msg_iov);
db8dac20
DM
1278
1279 if (err == -EINVAL)
1280 goto csum_copy_err;
1281 }
1282
22911fc5
ED
1283 if (unlikely(err)) {
1284 trace_kfree_skb(skb, udp_recvmsg);
979402b1
ED
1285 if (!peeked) {
1286 atomic_inc(&sk->sk_drops);
1287 UDP_INC_STATS_USER(sock_net(sk),
1288 UDP_MIB_INERRORS, is_udplite);
1289 }
db8dac20 1290 goto out_free;
22911fc5 1291 }
db8dac20
DM
1292
1293 if (!peeked)
629ca23c
PE
1294 UDP_INC_STATS_USER(sock_net(sk),
1295 UDP_MIB_INDATAGRAMS, is_udplite);
db8dac20 1296
3b885787 1297 sock_recv_ts_and_drops(msg, sk, skb);
db8dac20
DM
1298
1299 /* Copy the address. */
c482c568 1300 if (sin) {
db8dac20
DM
1301 sin->sin_family = AF_INET;
1302 sin->sin_port = udp_hdr(skb)->source;
1303 sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1304 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
1305 }
1306 if (inet->cmsg_flags)
1307 ip_cmsg_recv(msg, skb);
1308
59c2cdae 1309 err = copied;
db8dac20
DM
1310 if (flags & MSG_TRUNC)
1311 err = ulen;
1312
1313out_free:
9d410c79 1314 skb_free_datagram_locked(sk, skb);
db8dac20
DM
1315out:
1316 return err;
1317
1318csum_copy_err:
8a74ad60 1319 slow = lock_sock_fast(sk);
6a5dc9e5
ED
1320 if (!skb_kill_datagram(sk, skb, flags)) {
1321 UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
629ca23c 1322 UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
6a5dc9e5 1323 }
8a74ad60 1324 unlock_sock_fast(sk, slow);
db8dac20
DM
1325
1326 if (noblock)
1327 return -EAGAIN;
9cfaa8de
XZ
1328
1329 /* starting over for a new packet */
1330 msg->msg_flags &= ~MSG_TRUNC;
db8dac20
DM
1331 goto try_again;
1332}
1333
1334
1da177e4
LT
1335int udp_disconnect(struct sock *sk, int flags)
1336{
1337 struct inet_sock *inet = inet_sk(sk);
1338 /*
1339 * 1003.1g - break association.
1340 */
e905a9ed 1341
1da177e4 1342 sk->sk_state = TCP_CLOSE;
c720c7e8
ED
1343 inet->inet_daddr = 0;
1344 inet->inet_dport = 0;
bdeab991 1345 sock_rps_reset_rxhash(sk);
1da177e4
LT
1346 sk->sk_bound_dev_if = 0;
1347 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1348 inet_reset_saddr(sk);
1349
1350 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1351 sk->sk_prot->unhash(sk);
c720c7e8 1352 inet->inet_sport = 0;
1da177e4
LT
1353 }
1354 sk_dst_reset(sk);
1355 return 0;
1356}
c482c568 1357EXPORT_SYMBOL(udp_disconnect);
1da177e4 1358
645ca708
ED
1359void udp_lib_unhash(struct sock *sk)
1360{
723b4610
ED
1361 if (sk_hashed(sk)) {
1362 struct udp_table *udptable = sk->sk_prot->h.udp_table;
512615b6
ED
1363 struct udp_hslot *hslot, *hslot2;
1364
1365 hslot = udp_hashslot(udptable, sock_net(sk),
1366 udp_sk(sk)->udp_port_hash);
1367 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
645ca708 1368
723b4610
ED
1369 spin_lock_bh(&hslot->lock);
1370 if (sk_nulls_del_node_init_rcu(sk)) {
fdcc8aa9 1371 hslot->count--;
c720c7e8 1372 inet_sk(sk)->inet_num = 0;
723b4610 1373 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
512615b6
ED
1374
1375 spin_lock(&hslot2->lock);
1376 hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1377 hslot2->count--;
1378 spin_unlock(&hslot2->lock);
723b4610
ED
1379 }
1380 spin_unlock_bh(&hslot->lock);
645ca708 1381 }
645ca708
ED
1382}
1383EXPORT_SYMBOL(udp_lib_unhash);
1384
719f8358
ED
1385/*
1386 * inet_rcv_saddr was changed, we must rehash secondary hash
1387 */
1388void udp_lib_rehash(struct sock *sk, u16 newhash)
1389{
1390 if (sk_hashed(sk)) {
1391 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1392 struct udp_hslot *hslot, *hslot2, *nhslot2;
1393
1394 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1395 nhslot2 = udp_hashslot2(udptable, newhash);
1396 udp_sk(sk)->udp_portaddr_hash = newhash;
1397 if (hslot2 != nhslot2) {
1398 hslot = udp_hashslot(udptable, sock_net(sk),
1399 udp_sk(sk)->udp_port_hash);
1400 /* we must lock primary chain too */
1401 spin_lock_bh(&hslot->lock);
1402
1403 spin_lock(&hslot2->lock);
1404 hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1405 hslot2->count--;
1406 spin_unlock(&hslot2->lock);
1407
1408 spin_lock(&nhslot2->lock);
1409 hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
1410 &nhslot2->head);
1411 nhslot2->count++;
1412 spin_unlock(&nhslot2->lock);
1413
1414 spin_unlock_bh(&hslot->lock);
1415 }
1416 }
1417}
1418EXPORT_SYMBOL(udp_lib_rehash);
1419
1420static void udp_v4_rehash(struct sock *sk)
1421{
1422 u16 new_hash = udp4_portaddr_hash(sock_net(sk),
1423 inet_sk(sk)->inet_rcv_saddr,
1424 inet_sk(sk)->inet_num);
1425 udp_lib_rehash(sk, new_hash);
1426}
1427
93821778
HX
1428static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1429{
fec5e652 1430 int rc;
766e9037 1431
005ec974 1432 if (inet_sk(sk)->inet_daddr) {
bdeab991 1433 sock_rps_save_rxhash(sk, skb);
005ec974
SB
1434 sk_mark_napi_id(sk, skb);
1435 }
fec5e652 1436
d826eb14 1437 rc = sock_queue_rcv_skb(sk, skb);
766e9037
ED
1438 if (rc < 0) {
1439 int is_udplite = IS_UDPLITE(sk);
93821778 1440
93821778 1441 /* Note that an ENOMEM error is charged twice */
766e9037 1442 if (rc == -ENOMEM)
93821778
HX
1443 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1444 is_udplite);
766e9037
ED
1445 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1446 kfree_skb(skb);
296f7ea7 1447 trace_udp_fail_queue_rcv_skb(rc, sk);
766e9037 1448 return -1;
93821778
HX
1449 }
1450
1451 return 0;
1452
93821778
HX
1453}
1454
447167bf
ED
1455static struct static_key udp_encap_needed __read_mostly;
1456void udp_encap_enable(void)
1457{
1458 if (!static_key_enabled(&udp_encap_needed))
1459 static_key_slow_inc(&udp_encap_needed);
1460}
1461EXPORT_SYMBOL(udp_encap_enable);
1462
db8dac20
DM
1463/* returns:
1464 * -1: error
1465 * 0: success
1466 * >0: "udp encap" protocol resubmission
1467 *
1468 * Note that in the success and error cases, the skb is assumed to
1469 * have either been requeued or freed.
1470 */
c482c568 1471int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
db8dac20
DM
1472{
1473 struct udp_sock *up = udp_sk(sk);
1474 int rc;
1475 int is_udplite = IS_UDPLITE(sk);
1476
1477 /*
1478 * Charge it to the socket, dropping if the queue is full.
1479 */
1480 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1481 goto drop;
1482 nf_reset(skb);
1483
447167bf 1484 if (static_key_false(&udp_encap_needed) && up->encap_type) {
0ad92ad0
ED
1485 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
1486
db8dac20
DM
1487 /*
1488 * This is an encapsulation socket so pass the skb to
1489 * the socket's udp_encap_rcv() hook. Otherwise, just
1490 * fall through and pass this up the UDP socket.
1491 * up->encap_rcv() returns the following value:
1492 * =0 if skb was successfully passed to the encap
1493 * handler or was discarded by it.
1494 * >0 if skb should be passed on to UDP.
1495 * <0 if skb should be resubmitted as proto -N
1496 */
1497
1498 /* if we're overly short, let UDP handle it */
0ad92ad0
ED
1499 encap_rcv = ACCESS_ONCE(up->encap_rcv);
1500 if (skb->len > sizeof(struct udphdr) && encap_rcv != NULL) {
db8dac20
DM
1501 int ret;
1502
0ad92ad0 1503 ret = encap_rcv(sk, skb);
db8dac20 1504 if (ret <= 0) {
0283328e
PE
1505 UDP_INC_STATS_BH(sock_net(sk),
1506 UDP_MIB_INDATAGRAMS,
db8dac20
DM
1507 is_udplite);
1508 return -ret;
1509 }
1510 }
1511
1512 /* FALLTHROUGH -- it's a UDP Packet */
1513 }
1514
1515 /*
1516 * UDP-Lite specific tests, ignored on UDP sockets
1517 */
1518 if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
1519
1520 /*
1521 * MIB statistics other than incrementing the error count are
1522 * disabled for the following two types of errors: these depend
1523 * on the application settings, not on the functioning of the
1524 * protocol stack as such.
1525 *
1526 * RFC 3828 here recommends (sec 3.3): "There should also be a
1527 * way ... to ... at least let the receiving application block
1528 * delivery of packets with coverage values less than a value
1529 * provided by the application."
1530 */
1531 if (up->pcrlen == 0) { /* full coverage was set */
afd46503
JP
1532 LIMIT_NETDEBUG(KERN_WARNING "UDPLite: partial coverage %d while full coverage %d requested\n",
1533 UDP_SKB_CB(skb)->cscov, skb->len);
db8dac20
DM
1534 goto drop;
1535 }
1536 /* The next case involves violating the min. coverage requested
1537 * by the receiver. This is subtle: if receiver wants x and x is
1538 * greater than the buffersize/MTU then receiver will complain
1539 * that it wants x while sender emits packets of smaller size y.
1540 * Therefore the above ...()->partial_cov statement is essential.
1541 */
1542 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
afd46503
JP
1543 LIMIT_NETDEBUG(KERN_WARNING "UDPLite: coverage %d too small, need min %d\n",
1544 UDP_SKB_CB(skb)->cscov, up->pcrlen);
db8dac20
DM
1545 goto drop;
1546 }
1547 }
1548
33d480ce
ED
1549 if (rcu_access_pointer(sk->sk_filter) &&
1550 udp_lib_checksum_complete(skb))
6a5dc9e5 1551 goto csum_error;
db8dac20 1552
c377411f 1553
f545a38f 1554 if (sk_rcvqueues_full(sk, skb, sk->sk_rcvbuf))
c377411f
ED
1555 goto drop;
1556
93821778 1557 rc = 0;
db8dac20 1558
fbf8866d 1559 ipv4_pktinfo_prepare(sk, skb);
93821778
HX
1560 bh_lock_sock(sk);
1561 if (!sock_owned_by_user(sk))
1562 rc = __udp_queue_rcv_skb(sk, skb);
f545a38f 1563 else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
55349790
ZY
1564 bh_unlock_sock(sk);
1565 goto drop;
1566 }
93821778
HX
1567 bh_unlock_sock(sk);
1568
1569 return rc;
db8dac20 1570
6a5dc9e5
ED
1571csum_error:
1572 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
db8dac20 1573drop:
0283328e 1574 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
8edf19c2 1575 atomic_inc(&sk->sk_drops);
db8dac20
DM
1576 kfree_skb(skb);
1577 return -1;
1578}
1579
1240d137
ED
1580
1581static void flush_stack(struct sock **stack, unsigned int count,
1582 struct sk_buff *skb, unsigned int final)
1583{
1584 unsigned int i;
1585 struct sk_buff *skb1 = NULL;
f6b8f32c 1586 struct sock *sk;
1240d137
ED
1587
1588 for (i = 0; i < count; i++) {
f6b8f32c 1589 sk = stack[i];
1240d137
ED
1590 if (likely(skb1 == NULL))
1591 skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
1592
f6b8f32c
ED
1593 if (!skb1) {
1594 atomic_inc(&sk->sk_drops);
1595 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1596 IS_UDPLITE(sk));
1597 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
1598 IS_UDPLITE(sk));
1599 }
1600
1601 if (skb1 && udp_queue_rcv_skb(sk, skb1) <= 0)
1240d137
ED
1602 skb1 = NULL;
1603 }
1604 if (unlikely(skb1))
1605 kfree_skb(skb1);
1606}
1607
421b3885
SB
1608static void udp_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
1609{
1610 struct dst_entry *dst = skb_dst(skb);
1611
1612 dst_hold(dst);
1613 sk->sk_rx_dst = dst;
1614}
1615
db8dac20
DM
1616/*
1617 * Multicasts and broadcasts go to each listener.
1618 *
1240d137 1619 * Note: called only from the BH handler context.
db8dac20 1620 */
e3163493 1621static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
db8dac20
DM
1622 struct udphdr *uh,
1623 __be32 saddr, __be32 daddr,
645ca708 1624 struct udp_table *udptable)
db8dac20 1625{
1240d137 1626 struct sock *sk, *stack[256 / sizeof(struct sock *)];
f86dcc5a 1627 struct udp_hslot *hslot = udp_hashslot(udptable, net, ntohs(uh->dest));
db8dac20 1628 int dif;
1240d137 1629 unsigned int i, count = 0;
db8dac20 1630
645ca708 1631 spin_lock(&hslot->lock);
88ab1932 1632 sk = sk_nulls_head(&hslot->head);
db8dac20 1633 dif = skb->dev->ifindex;
920a4611 1634 sk = udp_v4_mcast_next(net, sk, uh->dest, daddr, uh->source, saddr, dif);
1240d137
ED
1635 while (sk) {
1636 stack[count++] = sk;
1637 sk = udp_v4_mcast_next(net, sk_nulls_next(sk), uh->dest,
1638 daddr, uh->source, saddr, dif);
1639 if (unlikely(count == ARRAY_SIZE(stack))) {
1640 if (!sk)
1641 break;
1642 flush_stack(stack, count, skb, ~0);
1643 count = 0;
1644 }
1645 }
1646 /*
1647 * before releasing chain lock, we must take a reference on sockets
1648 */
1649 for (i = 0; i < count; i++)
1650 sock_hold(stack[i]);
1651
645ca708 1652 spin_unlock(&hslot->lock);
1240d137
ED
1653
1654 /*
1655 * do the slow work with no lock held
1656 */
1657 if (count) {
1658 flush_stack(stack, count, skb, count - 1);
1659
1660 for (i = 0; i < count; i++)
1661 sock_put(stack[i]);
1662 } else {
1663 kfree_skb(skb);
1664 }
db8dac20
DM
1665 return 0;
1666}
1667
1668/* Initialize UDP checksum. If exited with zero value (success),
1669 * CHECKSUM_UNNECESSARY means, that no more checks are required.
1670 * Otherwise, csum completion requires chacksumming packet body,
1671 * including udp header and folding it to skb->csum.
1672 */
1673static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
1674 int proto)
1675{
1676 const struct iphdr *iph;
1677 int err;
1678
1679 UDP_SKB_CB(skb)->partial_cov = 0;
1680 UDP_SKB_CB(skb)->cscov = skb->len;
1681
1682 if (proto == IPPROTO_UDPLITE) {
1683 err = udplite_checksum_init(skb, uh);
1684 if (err)
1685 return err;
1686 }
1687
1688 iph = ip_hdr(skb);
1689 if (uh->check == 0) {
1690 skb->ip_summed = CHECKSUM_UNNECESSARY;
1691 } else if (skb->ip_summed == CHECKSUM_COMPLETE) {
c482c568 1692 if (!csum_tcpudp_magic(iph->saddr, iph->daddr, skb->len,
db8dac20
DM
1693 proto, skb->csum))
1694 skb->ip_summed = CHECKSUM_UNNECESSARY;
1695 }
1696 if (!skb_csum_unnecessary(skb))
1697 skb->csum = csum_tcpudp_nofold(iph->saddr, iph->daddr,
1698 skb->len, proto, 0);
1699 /* Probably, we should checksum udp header (it should be in cache
1700 * in any case) and data in tiny packets (< rx copybreak).
1701 */
1702
1703 return 0;
1704}
1705
1706/*
1707 * All we need to do is get the socket, and then do a checksum.
1708 */
1709
645ca708 1710int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
db8dac20
DM
1711 int proto)
1712{
1713 struct sock *sk;
7b5e56f9 1714 struct udphdr *uh;
db8dac20 1715 unsigned short ulen;
adf30907 1716 struct rtable *rt = skb_rtable(skb);
2783ef23 1717 __be32 saddr, daddr;
0283328e 1718 struct net *net = dev_net(skb->dev);
db8dac20
DM
1719
1720 /*
1721 * Validate the packet.
1722 */
1723 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
1724 goto drop; /* No space for header. */
1725
7b5e56f9 1726 uh = udp_hdr(skb);
db8dac20 1727 ulen = ntohs(uh->len);
ccc2d97c
BM
1728 saddr = ip_hdr(skb)->saddr;
1729 daddr = ip_hdr(skb)->daddr;
1730
db8dac20
DM
1731 if (ulen > skb->len)
1732 goto short_packet;
1733
1734 if (proto == IPPROTO_UDP) {
1735 /* UDP validates ulen. */
1736 if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
1737 goto short_packet;
1738 uh = udp_hdr(skb);
1739 }
1740
1741 if (udp4_csum_init(skb, uh, proto))
1742 goto csum_error;
1743
421b3885
SB
1744 if (skb->sk) {
1745 int ret;
1746 sk = skb->sk;
1747
1748 if (unlikely(sk->sk_rx_dst == NULL))
1749 udp_sk_rx_dst_set(sk, skb);
db8dac20 1750
421b3885
SB
1751 ret = udp_queue_rcv_skb(sk, skb);
1752
1753 /* a return value > 0 means to resubmit the input, but
1754 * it wants the return to be -protocol, or 0
1755 */
1756 if (ret > 0)
1757 return -ret;
1758 return 0;
1759 } else {
1760 if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
1761 return __udp4_lib_mcast_deliver(net, skb, uh,
1762 saddr, daddr, udptable);
1763
1764 sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
1765 }
db8dac20
DM
1766
1767 if (sk != NULL) {
a5b50476
ET
1768 int ret;
1769
a5b50476 1770 ret = udp_queue_rcv_skb(sk, skb);
db8dac20
DM
1771 sock_put(sk);
1772
1773 /* a return value > 0 means to resubmit the input, but
1774 * it wants the return to be -protocol, or 0
1775 */
1776 if (ret > 0)
1777 return -ret;
1778 return 0;
1779 }
1780
1781 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
1782 goto drop;
1783 nf_reset(skb);
1784
1785 /* No socket. Drop packet silently, if checksum is wrong */
1786 if (udp_lib_checksum_complete(skb))
1787 goto csum_error;
1788
0283328e 1789 UDP_INC_STATS_BH(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
db8dac20
DM
1790 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
1791
1792 /*
1793 * Hmm. We got an UDP packet to a port to which we
1794 * don't wanna listen. Ignore it.
1795 */
1796 kfree_skb(skb);
1797 return 0;
1798
1799short_packet:
673d57e7 1800 LIMIT_NETDEBUG(KERN_DEBUG "UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
afd46503
JP
1801 proto == IPPROTO_UDPLITE ? "Lite" : "",
1802 &saddr, ntohs(uh->source),
1803 ulen, skb->len,
1804 &daddr, ntohs(uh->dest));
db8dac20
DM
1805 goto drop;
1806
1807csum_error:
1808 /*
1809 * RFC1122: OK. Discards the bad packet silently (as far as
1810 * the network is concerned, anyway) as per 4.1.3.4 (MUST).
1811 */
673d57e7 1812 LIMIT_NETDEBUG(KERN_DEBUG "UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
afd46503
JP
1813 proto == IPPROTO_UDPLITE ? "Lite" : "",
1814 &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
db8dac20 1815 ulen);
6a5dc9e5 1816 UDP_INC_STATS_BH(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
db8dac20 1817drop:
0283328e 1818 UDP_INC_STATS_BH(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
db8dac20
DM
1819 kfree_skb(skb);
1820 return 0;
1821}
1822
421b3885
SB
1823/* We can only early demux multicast if there is a single matching socket.
1824 * If more than one socket found returns NULL
1825 */
1826static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
1827 __be16 loc_port, __be32 loc_addr,
1828 __be16 rmt_port, __be32 rmt_addr,
1829 int dif)
1830{
1831 struct sock *sk, *result;
1832 struct hlist_nulls_node *node;
1833 unsigned short hnum = ntohs(loc_port);
1834 unsigned int count, slot = udp_hashfn(net, hnum, udp_table.mask);
1835 struct udp_hslot *hslot = &udp_table.hash[slot];
1836
1837 rcu_read_lock();
1838begin:
1839 count = 0;
1840 result = NULL;
1841 sk_nulls_for_each_rcu(sk, node, &hslot->head) {
1842 if (__udp_is_mcast_sock(net, sk,
1843 loc_port, loc_addr,
1844 rmt_port, rmt_addr,
1845 dif, hnum)) {
1846 result = sk;
1847 ++count;
1848 }
1849 }
1850 /*
1851 * if the nulls value we got at the end of this lookup is
1852 * not the expected one, we must restart lookup.
1853 * We probably met an item that was moved to another chain.
1854 */
1855 if (get_nulls_value(node) != slot)
1856 goto begin;
1857
1858 if (result) {
1859 if (count != 1 ||
1860 unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
1861 result = NULL;
f69b923a 1862 else if (unlikely(!__udp_is_mcast_sock(net, result,
421b3885
SB
1863 loc_port, loc_addr,
1864 rmt_port, rmt_addr,
1865 dif, hnum))) {
1866 sock_put(result);
1867 result = NULL;
1868 }
1869 }
1870 rcu_read_unlock();
1871 return result;
1872}
1873
1874/* For unicast we should only early demux connected sockets or we can
1875 * break forwarding setups. The chains here can be long so only check
1876 * if the first socket is an exact match and if not move on.
1877 */
1878static struct sock *__udp4_lib_demux_lookup(struct net *net,
1879 __be16 loc_port, __be32 loc_addr,
1880 __be16 rmt_port, __be32 rmt_addr,
1881 int dif)
1882{
1883 struct sock *sk, *result;
1884 struct hlist_nulls_node *node;
1885 unsigned short hnum = ntohs(loc_port);
1886 unsigned int hash2 = udp4_portaddr_hash(net, loc_addr, hnum);
1887 unsigned int slot2 = hash2 & udp_table.mask;
1888 struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
1889 INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr)
1890 const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
1891
1892 rcu_read_lock();
1893 result = NULL;
1894 udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
1895 if (INET_MATCH(sk, net, acookie,
1896 rmt_addr, loc_addr, ports, dif))
1897 result = sk;
1898 /* Only check first socket in chain */
1899 break;
1900 }
1901
1902 if (result) {
1903 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
1904 result = NULL;
1905 else if (unlikely(!INET_MATCH(sk, net, acookie,
1906 rmt_addr, loc_addr,
1907 ports, dif))) {
1908 sock_put(result);
1909 result = NULL;
1910 }
1911 }
1912 rcu_read_unlock();
1913 return result;
1914}
1915
1916void udp_v4_early_demux(struct sk_buff *skb)
1917{
1918 const struct iphdr *iph = ip_hdr(skb);
1919 const struct udphdr *uh = udp_hdr(skb);
1920 struct sock *sk;
1921 struct dst_entry *dst;
1922 struct net *net = dev_net(skb->dev);
1923 int dif = skb->dev->ifindex;
1924
1925 /* validate the packet */
1926 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
1927 return;
1928
1929 if (skb->pkt_type == PACKET_BROADCAST ||
1930 skb->pkt_type == PACKET_MULTICAST)
1931 sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
1932 uh->source, iph->saddr, dif);
1933 else if (skb->pkt_type == PACKET_HOST)
1934 sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
1935 uh->source, iph->saddr, dif);
1936 else
1937 return;
1938
1939 if (!sk)
1940 return;
1941
1942 skb->sk = sk;
1943 skb->destructor = sock_edemux;
1944 dst = sk->sk_rx_dst;
1945
1946 if (dst)
1947 dst = dst_check(dst, 0);
1948 if (dst)
1949 skb_dst_set_noref(skb, dst);
1950}
1951
db8dac20
DM
1952int udp_rcv(struct sk_buff *skb)
1953{
645ca708 1954 return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP);
db8dac20
DM
1955}
1956
7d06b2e0 1957void udp_destroy_sock(struct sock *sk)
db8dac20 1958{
44046a59 1959 struct udp_sock *up = udp_sk(sk);
8a74ad60 1960 bool slow = lock_sock_fast(sk);
db8dac20 1961 udp_flush_pending_frames(sk);
8a74ad60 1962 unlock_sock_fast(sk, slow);
44046a59
TP
1963 if (static_key_false(&udp_encap_needed) && up->encap_type) {
1964 void (*encap_destroy)(struct sock *sk);
1965 encap_destroy = ACCESS_ONCE(up->encap_destroy);
1966 if (encap_destroy)
1967 encap_destroy(sk);
1968 }
db8dac20
DM
1969}
1970
1da177e4
LT
1971/*
1972 * Socket option code for UDP
1973 */
4c0a6cb0 1974int udp_lib_setsockopt(struct sock *sk, int level, int optname,
b7058842 1975 char __user *optval, unsigned int optlen,
4c0a6cb0 1976 int (*push_pending_frames)(struct sock *))
1da177e4
LT
1977{
1978 struct udp_sock *up = udp_sk(sk);
1979 int val;
1980 int err = 0;
b2bf1e26 1981 int is_udplite = IS_UDPLITE(sk);
1da177e4 1982
c482c568 1983 if (optlen < sizeof(int))
1da177e4
LT
1984 return -EINVAL;
1985
1986 if (get_user(val, (int __user *)optval))
1987 return -EFAULT;
1988
6516c655 1989 switch (optname) {
1da177e4
LT
1990 case UDP_CORK:
1991 if (val != 0) {
1992 up->corkflag = 1;
1993 } else {
1994 up->corkflag = 0;
1995 lock_sock(sk);
4c0a6cb0 1996 (*push_pending_frames)(sk);
1da177e4
LT
1997 release_sock(sk);
1998 }
1999 break;
e905a9ed 2000
1da177e4
LT
2001 case UDP_ENCAP:
2002 switch (val) {
2003 case 0:
2004 case UDP_ENCAP_ESPINUDP:
2005 case UDP_ENCAP_ESPINUDP_NON_IKE:
067b207b
JC
2006 up->encap_rcv = xfrm4_udp_encap_rcv;
2007 /* FALLTHROUGH */
342f0234 2008 case UDP_ENCAP_L2TPINUDP:
1da177e4 2009 up->encap_type = val;
447167bf 2010 udp_encap_enable();
1da177e4
LT
2011 break;
2012 default:
2013 err = -ENOPROTOOPT;
2014 break;
2015 }
2016 break;
2017
ba4e58ec
GR
2018 /*
2019 * UDP-Lite's partial checksum coverage (RFC 3828).
2020 */
2021 /* The sender sets actual checksum coverage length via this option.
2022 * The case coverage > packet length is handled by send module. */
2023 case UDPLITE_SEND_CSCOV:
b2bf1e26 2024 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR
2025 return -ENOPROTOOPT;
2026 if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
2027 val = 8;
4be929be
AD
2028 else if (val > USHRT_MAX)
2029 val = USHRT_MAX;
ba4e58ec
GR
2030 up->pcslen = val;
2031 up->pcflag |= UDPLITE_SEND_CC;
2032 break;
2033
e905a9ed
YH
2034 /* The receiver specifies a minimum checksum coverage value. To make
2035 * sense, this should be set to at least 8 (as done below). If zero is
ba4e58ec
GR
2036 * used, this again means full checksum coverage. */
2037 case UDPLITE_RECV_CSCOV:
b2bf1e26 2038 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR
2039 return -ENOPROTOOPT;
2040 if (val != 0 && val < 8) /* Avoid silly minimal values. */
2041 val = 8;
4be929be
AD
2042 else if (val > USHRT_MAX)
2043 val = USHRT_MAX;
ba4e58ec
GR
2044 up->pcrlen = val;
2045 up->pcflag |= UDPLITE_RECV_CC;
2046 break;
2047
1da177e4
LT
2048 default:
2049 err = -ENOPROTOOPT;
2050 break;
6516c655 2051 }
1da177e4
LT
2052
2053 return err;
2054}
c482c568 2055EXPORT_SYMBOL(udp_lib_setsockopt);
1da177e4 2056
db8dac20 2057int udp_setsockopt(struct sock *sk, int level, int optname,
b7058842 2058 char __user *optval, unsigned int optlen)
db8dac20
DM
2059{
2060 if (level == SOL_UDP || level == SOL_UDPLITE)
2061 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2062 udp_push_pending_frames);
2063 return ip_setsockopt(sk, level, optname, optval, optlen);
2064}
2065
2066#ifdef CONFIG_COMPAT
2067int compat_udp_setsockopt(struct sock *sk, int level, int optname,
b7058842 2068 char __user *optval, unsigned int optlen)
db8dac20
DM
2069{
2070 if (level == SOL_UDP || level == SOL_UDPLITE)
2071 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2072 udp_push_pending_frames);
2073 return compat_ip_setsockopt(sk, level, optname, optval, optlen);
2074}
2075#endif
2076
4c0a6cb0
GR
2077int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2078 char __user *optval, int __user *optlen)
1da177e4
LT
2079{
2080 struct udp_sock *up = udp_sk(sk);
2081 int val, len;
2082
c482c568 2083 if (get_user(len, optlen))
1da177e4
LT
2084 return -EFAULT;
2085
2086 len = min_t(unsigned int, len, sizeof(int));
e905a9ed 2087
6516c655 2088 if (len < 0)
1da177e4
LT
2089 return -EINVAL;
2090
6516c655 2091 switch (optname) {
1da177e4
LT
2092 case UDP_CORK:
2093 val = up->corkflag;
2094 break;
2095
2096 case UDP_ENCAP:
2097 val = up->encap_type;
2098 break;
2099
ba4e58ec
GR
2100 /* The following two cannot be changed on UDP sockets, the return is
2101 * always 0 (which corresponds to the full checksum coverage of UDP). */
2102 case UDPLITE_SEND_CSCOV:
2103 val = up->pcslen;
2104 break;
2105
2106 case UDPLITE_RECV_CSCOV:
2107 val = up->pcrlen;
2108 break;
2109
1da177e4
LT
2110 default:
2111 return -ENOPROTOOPT;
6516c655 2112 }
1da177e4 2113
6516c655 2114 if (put_user(len, optlen))
e905a9ed 2115 return -EFAULT;
c482c568 2116 if (copy_to_user(optval, &val, len))
1da177e4 2117 return -EFAULT;
e905a9ed 2118 return 0;
1da177e4 2119}
c482c568 2120EXPORT_SYMBOL(udp_lib_getsockopt);
1da177e4 2121
db8dac20
DM
2122int udp_getsockopt(struct sock *sk, int level, int optname,
2123 char __user *optval, int __user *optlen)
2124{
2125 if (level == SOL_UDP || level == SOL_UDPLITE)
2126 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2127 return ip_getsockopt(sk, level, optname, optval, optlen);
2128}
2129
2130#ifdef CONFIG_COMPAT
2131int compat_udp_getsockopt(struct sock *sk, int level, int optname,
2132 char __user *optval, int __user *optlen)
2133{
2134 if (level == SOL_UDP || level == SOL_UDPLITE)
2135 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2136 return compat_ip_getsockopt(sk, level, optname, optval, optlen);
2137}
2138#endif
1da177e4
LT
2139/**
2140 * udp_poll - wait for a UDP event.
2141 * @file - file struct
2142 * @sock - socket
2143 * @wait - poll table
2144 *
e905a9ed 2145 * This is same as datagram poll, except for the special case of
1da177e4
LT
2146 * blocking sockets. If application is using a blocking fd
2147 * and a packet with checksum error is in the queue;
2148 * then it could get return from select indicating data available
2149 * but then block when reading it. Add special case code
2150 * to work around these arguably broken applications.
2151 */
2152unsigned int udp_poll(struct file *file, struct socket *sock, poll_table *wait)
2153{
2154 unsigned int mask = datagram_poll(file, sock, wait);
2155 struct sock *sk = sock->sk;
ba4e58ec 2156
c3f1dbaf
DM
2157 sock_rps_record_flow(sk);
2158
1da177e4 2159 /* Check for false positives due to checksum errors */
85584672
ED
2160 if ((mask & POLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
2161 !(sk->sk_shutdown & RCV_SHUTDOWN) && !first_packet_length(sk))
2162 mask &= ~(POLLIN | POLLRDNORM);
1da177e4
LT
2163
2164 return mask;
e905a9ed 2165
1da177e4 2166}
c482c568 2167EXPORT_SYMBOL(udp_poll);
1da177e4 2168
db8dac20
DM
2169struct proto udp_prot = {
2170 .name = "UDP",
2171 .owner = THIS_MODULE,
2172 .close = udp_lib_close,
2173 .connect = ip4_datagram_connect,
2174 .disconnect = udp_disconnect,
2175 .ioctl = udp_ioctl,
2176 .destroy = udp_destroy_sock,
2177 .setsockopt = udp_setsockopt,
2178 .getsockopt = udp_getsockopt,
2179 .sendmsg = udp_sendmsg,
2180 .recvmsg = udp_recvmsg,
2181 .sendpage = udp_sendpage,
93821778 2182 .backlog_rcv = __udp_queue_rcv_skb,
8141ed9f 2183 .release_cb = ip4_datagram_release_cb,
db8dac20
DM
2184 .hash = udp_lib_hash,
2185 .unhash = udp_lib_unhash,
719f8358 2186 .rehash = udp_v4_rehash,
db8dac20
DM
2187 .get_port = udp_v4_get_port,
2188 .memory_allocated = &udp_memory_allocated,
2189 .sysctl_mem = sysctl_udp_mem,
2190 .sysctl_wmem = &sysctl_udp_wmem_min,
2191 .sysctl_rmem = &sysctl_udp_rmem_min,
2192 .obj_size = sizeof(struct udp_sock),
271b72c7 2193 .slab_flags = SLAB_DESTROY_BY_RCU,
645ca708 2194 .h.udp_table = &udp_table,
db8dac20
DM
2195#ifdef CONFIG_COMPAT
2196 .compat_setsockopt = compat_udp_setsockopt,
2197 .compat_getsockopt = compat_udp_getsockopt,
2198#endif
fcbdf09d 2199 .clear_sk = sk_prot_clear_portaddr_nulls,
db8dac20 2200};
c482c568 2201EXPORT_SYMBOL(udp_prot);
1da177e4
LT
2202
2203/* ------------------------------------------------------------------------ */
2204#ifdef CONFIG_PROC_FS
2205
645ca708 2206static struct sock *udp_get_first(struct seq_file *seq, int start)
1da177e4
LT
2207{
2208 struct sock *sk;
2209 struct udp_iter_state *state = seq->private;
6f191efe 2210 struct net *net = seq_file_net(seq);
1da177e4 2211
f86dcc5a
ED
2212 for (state->bucket = start; state->bucket <= state->udp_table->mask;
2213 ++state->bucket) {
88ab1932 2214 struct hlist_nulls_node *node;
645ca708 2215 struct udp_hslot *hslot = &state->udp_table->hash[state->bucket];
f86dcc5a
ED
2216
2217 if (hlist_nulls_empty(&hslot->head))
2218 continue;
2219
645ca708 2220 spin_lock_bh(&hslot->lock);
88ab1932 2221 sk_nulls_for_each(sk, node, &hslot->head) {
878628fb 2222 if (!net_eq(sock_net(sk), net))
a91275ef 2223 continue;
1da177e4
LT
2224 if (sk->sk_family == state->family)
2225 goto found;
2226 }
645ca708 2227 spin_unlock_bh(&hslot->lock);
1da177e4
LT
2228 }
2229 sk = NULL;
2230found:
2231 return sk;
2232}
2233
2234static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
2235{
2236 struct udp_iter_state *state = seq->private;
6f191efe 2237 struct net *net = seq_file_net(seq);
1da177e4
LT
2238
2239 do {
88ab1932 2240 sk = sk_nulls_next(sk);
878628fb 2241 } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family));
1da177e4 2242
645ca708 2243 if (!sk) {
f86dcc5a 2244 if (state->bucket <= state->udp_table->mask)
30842f29 2245 spin_unlock_bh(&state->udp_table->hash[state->bucket].lock);
645ca708 2246 return udp_get_first(seq, state->bucket + 1);
1da177e4
LT
2247 }
2248 return sk;
2249}
2250
2251static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
2252{
645ca708 2253 struct sock *sk = udp_get_first(seq, 0);
1da177e4
LT
2254
2255 if (sk)
6516c655 2256 while (pos && (sk = udp_get_next(seq, sk)) != NULL)
1da177e4
LT
2257 --pos;
2258 return pos ? NULL : sk;
2259}
2260
2261static void *udp_seq_start(struct seq_file *seq, loff_t *pos)
2262{
30842f29 2263 struct udp_iter_state *state = seq->private;
f86dcc5a 2264 state->bucket = MAX_UDP_PORTS;
30842f29 2265
b50660f1 2266 return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
1da177e4
LT
2267}
2268
2269static void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2270{
2271 struct sock *sk;
2272
b50660f1 2273 if (v == SEQ_START_TOKEN)
1da177e4
LT
2274 sk = udp_get_idx(seq, 0);
2275 else
2276 sk = udp_get_next(seq, v);
2277
2278 ++*pos;
2279 return sk;
2280}
2281
2282static void udp_seq_stop(struct seq_file *seq, void *v)
2283{
645ca708
ED
2284 struct udp_iter_state *state = seq->private;
2285
f86dcc5a 2286 if (state->bucket <= state->udp_table->mask)
645ca708 2287 spin_unlock_bh(&state->udp_table->hash[state->bucket].lock);
1da177e4
LT
2288}
2289
73cb88ec 2290int udp_seq_open(struct inode *inode, struct file *file)
1da177e4 2291{
d9dda78b 2292 struct udp_seq_afinfo *afinfo = PDE_DATA(inode);
a2be75c1
DL
2293 struct udp_iter_state *s;
2294 int err;
a91275ef 2295
a2be75c1
DL
2296 err = seq_open_net(inode, file, &afinfo->seq_ops,
2297 sizeof(struct udp_iter_state));
2298 if (err < 0)
2299 return err;
a91275ef 2300
a2be75c1 2301 s = ((struct seq_file *)file->private_data)->private;
1da177e4 2302 s->family = afinfo->family;
645ca708 2303 s->udp_table = afinfo->udp_table;
a2be75c1 2304 return err;
a91275ef 2305}
73cb88ec 2306EXPORT_SYMBOL(udp_seq_open);
a91275ef 2307
1da177e4 2308/* ------------------------------------------------------------------------ */
0c96d8c5 2309int udp_proc_register(struct net *net, struct udp_seq_afinfo *afinfo)
1da177e4
LT
2310{
2311 struct proc_dir_entry *p;
2312 int rc = 0;
2313
dda61925
DL
2314 afinfo->seq_ops.start = udp_seq_start;
2315 afinfo->seq_ops.next = udp_seq_next;
2316 afinfo->seq_ops.stop = udp_seq_stop;
2317
84841c3c 2318 p = proc_create_data(afinfo->name, S_IRUGO, net->proc_net,
73cb88ec 2319 afinfo->seq_fops, afinfo);
84841c3c 2320 if (!p)
1da177e4
LT
2321 rc = -ENOMEM;
2322 return rc;
2323}
c482c568 2324EXPORT_SYMBOL(udp_proc_register);
1da177e4 2325
0c96d8c5 2326void udp_proc_unregister(struct net *net, struct udp_seq_afinfo *afinfo)
1da177e4 2327{
ece31ffd 2328 remove_proc_entry(afinfo->name, net->proc_net);
1da177e4 2329}
c482c568 2330EXPORT_SYMBOL(udp_proc_unregister);
db8dac20
DM
2331
2332/* ------------------------------------------------------------------------ */
5e659e4c
PE
2333static void udp4_format_sock(struct sock *sp, struct seq_file *f,
2334 int bucket, int *len)
db8dac20
DM
2335{
2336 struct inet_sock *inet = inet_sk(sp);
c720c7e8
ED
2337 __be32 dest = inet->inet_daddr;
2338 __be32 src = inet->inet_rcv_saddr;
2339 __u16 destp = ntohs(inet->inet_dport);
2340 __u16 srcp = ntohs(inet->inet_sport);
db8dac20 2341
f86dcc5a 2342 seq_printf(f, "%5d: %08X:%04X %08X:%04X"
d14c5ab6 2343 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d%n",
db8dac20 2344 bucket, src, srcp, dest, destp, sp->sk_state,
31e6d363
ED
2345 sk_wmem_alloc_get(sp),
2346 sk_rmem_alloc_get(sp),
a7cb5a49
EB
2347 0, 0L, 0,
2348 from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
2349 0, sock_i_ino(sp),
cb61cb9b
ED
2350 atomic_read(&sp->sk_refcnt), sp,
2351 atomic_read(&sp->sk_drops), len);
db8dac20
DM
2352}
2353
2354int udp4_seq_show(struct seq_file *seq, void *v)
2355{
2356 if (v == SEQ_START_TOKEN)
2357 seq_printf(seq, "%-127s\n",
2358 " sl local_address rem_address st tx_queue "
2359 "rx_queue tr tm->when retrnsmt uid timeout "
cb61cb9b 2360 "inode ref pointer drops");
db8dac20 2361 else {
db8dac20 2362 struct udp_iter_state *state = seq->private;
5e659e4c 2363 int len;
db8dac20 2364
5e659e4c 2365 udp4_format_sock(v, seq, state->bucket, &len);
c482c568 2366 seq_printf(seq, "%*s\n", 127 - len, "");
db8dac20
DM
2367 }
2368 return 0;
2369}
2370
73cb88ec
AV
2371static const struct file_operations udp_afinfo_seq_fops = {
2372 .owner = THIS_MODULE,
2373 .open = udp_seq_open,
2374 .read = seq_read,
2375 .llseek = seq_lseek,
2376 .release = seq_release_net
2377};
2378
db8dac20 2379/* ------------------------------------------------------------------------ */
db8dac20 2380static struct udp_seq_afinfo udp4_seq_afinfo = {
db8dac20
DM
2381 .name = "udp",
2382 .family = AF_INET,
645ca708 2383 .udp_table = &udp_table,
73cb88ec 2384 .seq_fops = &udp_afinfo_seq_fops,
dda61925
DL
2385 .seq_ops = {
2386 .show = udp4_seq_show,
2387 },
db8dac20
DM
2388};
2389
2c8c1e72 2390static int __net_init udp4_proc_init_net(struct net *net)
15439feb
PE
2391{
2392 return udp_proc_register(net, &udp4_seq_afinfo);
2393}
2394
2c8c1e72 2395static void __net_exit udp4_proc_exit_net(struct net *net)
15439feb
PE
2396{
2397 udp_proc_unregister(net, &udp4_seq_afinfo);
2398}
2399
2400static struct pernet_operations udp4_net_ops = {
2401 .init = udp4_proc_init_net,
2402 .exit = udp4_proc_exit_net,
2403};
2404
db8dac20
DM
2405int __init udp4_proc_init(void)
2406{
15439feb 2407 return register_pernet_subsys(&udp4_net_ops);
db8dac20
DM
2408}
2409
2410void udp4_proc_exit(void)
2411{
15439feb 2412 unregister_pernet_subsys(&udp4_net_ops);
db8dac20 2413}
1da177e4
LT
2414#endif /* CONFIG_PROC_FS */
2415
f86dcc5a
ED
2416static __initdata unsigned long uhash_entries;
2417static int __init set_uhash_entries(char *str)
645ca708 2418{
413c27d8
EZ
2419 ssize_t ret;
2420
f86dcc5a
ED
2421 if (!str)
2422 return 0;
413c27d8
EZ
2423
2424 ret = kstrtoul(str, 0, &uhash_entries);
2425 if (ret)
2426 return 0;
2427
f86dcc5a
ED
2428 if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
2429 uhash_entries = UDP_HTABLE_SIZE_MIN;
2430 return 1;
2431}
2432__setup("uhash_entries=", set_uhash_entries);
645ca708 2433
f86dcc5a
ED
2434void __init udp_table_init(struct udp_table *table, const char *name)
2435{
2436 unsigned int i;
2437
31fe62b9
TB
2438 table->hash = alloc_large_system_hash(name,
2439 2 * sizeof(struct udp_hslot),
2440 uhash_entries,
2441 21, /* one slot per 2 MB */
2442 0,
2443 &table->log,
2444 &table->mask,
2445 UDP_HTABLE_SIZE_MIN,
2446 64 * 1024);
2447
512615b6 2448 table->hash2 = table->hash + (table->mask + 1);
f86dcc5a 2449 for (i = 0; i <= table->mask; i++) {
88ab1932 2450 INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i);
fdcc8aa9 2451 table->hash[i].count = 0;
645ca708
ED
2452 spin_lock_init(&table->hash[i].lock);
2453 }
512615b6
ED
2454 for (i = 0; i <= table->mask; i++) {
2455 INIT_HLIST_NULLS_HEAD(&table->hash2[i].head, i);
2456 table->hash2[i].count = 0;
2457 spin_lock_init(&table->hash2[i].lock);
2458 }
645ca708
ED
2459}
2460
95766fff
HA
2461void __init udp_init(void)
2462{
f03d78db 2463 unsigned long limit;
95766fff 2464
f86dcc5a 2465 udp_table_init(&udp_table, "UDP");
f03d78db 2466 limit = nr_free_buffer_pages() / 8;
95766fff
HA
2467 limit = max(limit, 128UL);
2468 sysctl_udp_mem[0] = limit / 4 * 3;
2469 sysctl_udp_mem[1] = limit;
2470 sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
2471
2472 sysctl_udp_rmem_min = SK_MEM_QUANTUM;
2473 sysctl_udp_wmem_min = SK_MEM_QUANTUM;
2474}
2475
da5bab07
DB
2476struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
2477 netdev_features_t features)
73136267
PS
2478{
2479 struct sk_buff *segs = ERR_PTR(-EINVAL);
2480 int mac_len = skb->mac_len;
2481 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
0d05535d 2482 __be16 protocol = skb->protocol;
73136267 2483 netdev_features_t enc_features;
0d05535d 2484 int outer_hlen;
73136267
PS
2485
2486 if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
2487 goto out;
2488
2489 skb->encapsulation = 0;
2490 __skb_pull(skb, tnl_hlen);
2491 skb_reset_mac_header(skb);
2492 skb_set_network_header(skb, skb_inner_network_offset(skb));
2493 skb->mac_len = skb_inner_network_offset(skb);
19acc327 2494 skb->protocol = htons(ETH_P_TEB);
73136267
PS
2495
2496 /* segment inner packet. */
2497 enc_features = skb->dev->hw_enc_features & netif_skb_features(skb);
2498 segs = skb_mac_gso_segment(skb, enc_features);
2499 if (!segs || IS_ERR(segs))
2500 goto out;
2501
2502 outer_hlen = skb_tnl_header_len(skb);
2503 skb = segs;
2504 do {
2505 struct udphdr *uh;
2506 int udp_offset = outer_hlen - tnl_hlen;
2507
cdbaa0bb
AD
2508 skb_reset_inner_headers(skb);
2509 skb->encapsulation = 1;
2510
73136267
PS
2511 skb->mac_len = mac_len;
2512
2513 skb_push(skb, outer_hlen);
2514 skb_reset_mac_header(skb);
2515 skb_set_network_header(skb, mac_len);
2516 skb_set_transport_header(skb, udp_offset);
2517 uh = udp_hdr(skb);
2518 uh->len = htons(skb->len - udp_offset);
2519
2520 /* csum segment if tunnel sets skb with csum. */
eb3c0d83 2521 if (protocol == htons(ETH_P_IP) && unlikely(uh->check)) {
73136267
PS
2522 struct iphdr *iph = ip_hdr(skb);
2523
2524 uh->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
2525 skb->len - udp_offset,
2526 IPPROTO_UDP, 0);
2527 uh->check = csum_fold(skb_checksum(skb, udp_offset,
2528 skb->len - udp_offset, 0));
2529 if (uh->check == 0)
2530 uh->check = CSUM_MANGLED_0;
2531
eb3c0d83
CW
2532 } else if (protocol == htons(ETH_P_IPV6)) {
2533 struct ipv6hdr *ipv6h = ipv6_hdr(skb);
2534 u32 len = skb->len - udp_offset;
2535
2536 uh->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
2537 len, IPPROTO_UDP, 0);
2538 uh->check = csum_fold(skb_checksum(skb, udp_offset, len, 0));
2539 if (uh->check == 0)
2540 uh->check = CSUM_MANGLED_0;
2541 skb->ip_summed = CHECKSUM_NONE;
73136267 2542 }
eb3c0d83 2543
0d05535d 2544 skb->protocol = protocol;
73136267
PS
2545 } while ((skb = skb->next));
2546out:
2547 return segs;
2548}