2 * IPv6 output functions
3 * Linux INET6 implementation
6 * Pedro Roque <roque@di.fc.ul.pt>
8 * Based on linux/net/ipv4/ip_output.c
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
16 * A.N.Kuznetsov : airthmetics in fragmentation.
17 * extension headers are implemented.
18 * route changes now work.
19 * ip6_forward does not confuse sniffers.
22 * H. von Brand : Added missing #include <linux/string.h>
23 * Imran Patel : frag id should be in NBO
24 * Kazunori MIYAZAWA @USAGI
25 * : add ip6_append_data and related functions
29 #include <linux/errno.h>
30 #include <linux/kernel.h>
31 #include <linux/string.h>
32 #include <linux/socket.h>
33 #include <linux/net.h>
34 #include <linux/netdevice.h>
35 #include <linux/if_arp.h>
36 #include <linux/in6.h>
37 #include <linux/tcp.h>
38 #include <linux/route.h>
39 #include <linux/module.h>
40 #include <linux/slab.h>
42 #include <linux/bpf-cgroup.h>
43 #include <linux/netfilter.h>
44 #include <linux/netfilter_ipv6.h>
50 #include <net/ndisc.h>
51 #include <net/protocol.h>
52 #include <net/ip6_route.h>
53 #include <net/addrconf.h>
54 #include <net/rawv6.h>
57 #include <net/checksum.h>
58 #include <linux/mroute6.h>
59 #include <net/l3mdev.h>
60 #include <net/lwtunnel.h>
62 static int ip6_finish_output2(struct net
*net
, struct sock
*sk
, struct sk_buff
*skb
)
64 struct dst_entry
*dst
= skb_dst(skb
);
65 struct net_device
*dev
= dst
->dev
;
66 struct neighbour
*neigh
;
67 struct in6_addr
*nexthop
;
70 skb
->protocol
= htons(ETH_P_IPV6
);
73 if (ipv6_addr_is_multicast(&ipv6_hdr(skb
)->daddr
)) {
74 struct inet6_dev
*idev
= ip6_dst_idev(skb_dst(skb
));
76 if (!(dev
->flags
& IFF_LOOPBACK
) && sk_mc_loop(sk
) &&
77 ((mroute6_socket(net
, skb
) &&
78 !(IP6CB(skb
)->flags
& IP6SKB_FORWARDED
)) ||
79 ipv6_chk_mcast_addr(dev
, &ipv6_hdr(skb
)->daddr
,
80 &ipv6_hdr(skb
)->saddr
))) {
81 struct sk_buff
*newskb
= skb_clone(skb
, GFP_ATOMIC
);
83 /* Do not check for IFF_ALLMULTI; multicast routing
84 is not supported in any case.
87 NF_HOOK(NFPROTO_IPV6
, NF_INET_POST_ROUTING
,
88 net
, sk
, newskb
, NULL
, newskb
->dev
,
91 if (ipv6_hdr(skb
)->hop_limit
== 0) {
92 IP6_INC_STATS(net
, idev
,
93 IPSTATS_MIB_OUTDISCARDS
);
99 IP6_UPD_PO_STATS(net
, idev
, IPSTATS_MIB_OUTMCAST
, skb
->len
);
101 if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb
)->daddr
) <=
102 IPV6_ADDR_SCOPE_NODELOCAL
&&
103 !(dev
->flags
& IFF_LOOPBACK
)) {
109 if (lwtunnel_xmit_redirect(dst
->lwtstate
)) {
110 int res
= lwtunnel_xmit(skb
);
112 if (res
< 0 || res
== LWTUNNEL_XMIT_DONE
)
117 nexthop
= rt6_nexthop((struct rt6_info
*)dst
, &ipv6_hdr(skb
)->daddr
);
118 neigh
= __ipv6_neigh_lookup_noref(dst
->dev
, nexthop
);
119 if (unlikely(!neigh
))
120 neigh
= __neigh_create(&nd_tbl
, nexthop
, dst
->dev
, false);
121 if (!IS_ERR(neigh
)) {
122 ret
= dst_neigh_output(dst
, neigh
, skb
);
123 rcu_read_unlock_bh();
126 rcu_read_unlock_bh();
128 IP6_INC_STATS(net
, ip6_dst_idev(dst
), IPSTATS_MIB_OUTNOROUTES
);
133 static int ip6_finish_output(struct net
*net
, struct sock
*sk
, struct sk_buff
*skb
)
137 ret
= BPF_CGROUP_RUN_PROG_INET_EGRESS(sk
, skb
);
143 if ((skb
->len
> ip6_skb_dst_mtu(skb
) && !skb_is_gso(skb
)) ||
144 dst_allfrag(skb_dst(skb
)) ||
145 (IP6CB(skb
)->frag_max_size
&& skb
->len
> IP6CB(skb
)->frag_max_size
))
146 return ip6_fragment(net
, sk
, skb
, ip6_finish_output2
);
148 return ip6_finish_output2(net
, sk
, skb
);
151 int ip6_output(struct net
*net
, struct sock
*sk
, struct sk_buff
*skb
)
153 struct net_device
*dev
= skb_dst(skb
)->dev
;
154 struct inet6_dev
*idev
= ip6_dst_idev(skb_dst(skb
));
156 if (unlikely(idev
->cnf
.disable_ipv6
)) {
157 IP6_INC_STATS(net
, idev
, IPSTATS_MIB_OUTDISCARDS
);
162 return NF_HOOK_COND(NFPROTO_IPV6
, NF_INET_POST_ROUTING
,
163 net
, sk
, skb
, NULL
, dev
,
165 !(IP6CB(skb
)->flags
& IP6SKB_REROUTED
));
169 * xmit an sk_buff (used by TCP, SCTP and DCCP)
170 * Note : socket lock is not held for SYNACK packets, but might be modified
171 * by calls to skb_set_owner_w() and ipv6_local_error(),
172 * which are using proper atomic operations or spinlocks.
174 int ip6_xmit(const struct sock
*sk
, struct sk_buff
*skb
, struct flowi6
*fl6
,
175 __u32 mark
, struct ipv6_txoptions
*opt
, int tclass
)
177 struct net
*net
= sock_net(sk
);
178 const struct ipv6_pinfo
*np
= inet6_sk(sk
);
179 struct in6_addr
*first_hop
= &fl6
->daddr
;
180 struct dst_entry
*dst
= skb_dst(skb
);
182 u8 proto
= fl6
->flowi6_proto
;
183 int seg_len
= skb
->len
;
188 unsigned int head_room
;
190 /* First: exthdrs may take lots of space (~8K for now)
191 MAX_HEADER is not enough.
193 head_room
= opt
->opt_nflen
+ opt
->opt_flen
;
194 seg_len
+= head_room
;
195 head_room
+= sizeof(struct ipv6hdr
) + LL_RESERVED_SPACE(dst
->dev
);
197 if (skb_headroom(skb
) < head_room
) {
198 struct sk_buff
*skb2
= skb_realloc_headroom(skb
, head_room
);
200 IP6_INC_STATS(net
, ip6_dst_idev(skb_dst(skb
)),
201 IPSTATS_MIB_OUTDISCARDS
);
207 /* skb_set_owner_w() changes sk->sk_wmem_alloc atomically,
208 * it is safe to call in our context (socket lock not held)
210 skb_set_owner_w(skb
, (struct sock
*)sk
);
213 ipv6_push_frag_opts(skb
, opt
, &proto
);
215 ipv6_push_nfrag_opts(skb
, opt
, &proto
, &first_hop
,
219 skb_push(skb
, sizeof(struct ipv6hdr
));
220 skb_reset_network_header(skb
);
224 * Fill in the IPv6 header
227 hlimit
= np
->hop_limit
;
229 hlimit
= ip6_dst_hoplimit(dst
);
231 ip6_flow_hdr(hdr
, tclass
, ip6_make_flowlabel(net
, skb
, fl6
->flowlabel
,
232 np
->autoflowlabel
, fl6
));
234 hdr
->payload_len
= htons(seg_len
);
235 hdr
->nexthdr
= proto
;
236 hdr
->hop_limit
= hlimit
;
238 hdr
->saddr
= fl6
->saddr
;
239 hdr
->daddr
= *first_hop
;
241 skb
->protocol
= htons(ETH_P_IPV6
);
242 skb
->priority
= sk
->sk_priority
;
246 if ((skb
->len
<= mtu
) || skb
->ignore_df
|| skb_is_gso(skb
)) {
247 IP6_UPD_PO_STATS(net
, ip6_dst_idev(skb_dst(skb
)),
248 IPSTATS_MIB_OUT
, skb
->len
);
250 /* if egress device is enslaved to an L3 master device pass the
251 * skb to its handler for processing
253 skb
= l3mdev_ip6_out((struct sock
*)sk
, skb
);
257 /* hooks should never assume socket lock is held.
258 * we promote our socket to non const
260 return NF_HOOK(NFPROTO_IPV6
, NF_INET_LOCAL_OUT
,
261 net
, (struct sock
*)sk
, skb
, NULL
, dst
->dev
,
266 /* ipv6_local_error() does not require socket lock,
267 * we promote our socket to non const
269 ipv6_local_error((struct sock
*)sk
, EMSGSIZE
, fl6
, mtu
);
271 IP6_INC_STATS(net
, ip6_dst_idev(skb_dst(skb
)), IPSTATS_MIB_FRAGFAILS
);
275 EXPORT_SYMBOL(ip6_xmit
);
277 static int ip6_call_ra_chain(struct sk_buff
*skb
, int sel
)
279 struct ip6_ra_chain
*ra
;
280 struct sock
*last
= NULL
;
282 read_lock(&ip6_ra_lock
);
283 for (ra
= ip6_ra_chain
; ra
; ra
= ra
->next
) {
284 struct sock
*sk
= ra
->sk
;
285 if (sk
&& ra
->sel
== sel
&&
286 (!sk
->sk_bound_dev_if
||
287 sk
->sk_bound_dev_if
== skb
->dev
->ifindex
)) {
289 struct sk_buff
*skb2
= skb_clone(skb
, GFP_ATOMIC
);
291 rawv6_rcv(last
, skb2
);
298 rawv6_rcv(last
, skb
);
299 read_unlock(&ip6_ra_lock
);
302 read_unlock(&ip6_ra_lock
);
306 static int ip6_forward_proxy_check(struct sk_buff
*skb
)
308 struct ipv6hdr
*hdr
= ipv6_hdr(skb
);
309 u8 nexthdr
= hdr
->nexthdr
;
313 if (ipv6_ext_hdr(nexthdr
)) {
314 offset
= ipv6_skip_exthdr(skb
, sizeof(*hdr
), &nexthdr
, &frag_off
);
318 offset
= sizeof(struct ipv6hdr
);
320 if (nexthdr
== IPPROTO_ICMPV6
) {
321 struct icmp6hdr
*icmp6
;
323 if (!pskb_may_pull(skb
, (skb_network_header(skb
) +
324 offset
+ 1 - skb
->data
)))
327 icmp6
= (struct icmp6hdr
*)(skb_network_header(skb
) + offset
);
329 switch (icmp6
->icmp6_type
) {
330 case NDISC_ROUTER_SOLICITATION
:
331 case NDISC_ROUTER_ADVERTISEMENT
:
332 case NDISC_NEIGHBOUR_SOLICITATION
:
333 case NDISC_NEIGHBOUR_ADVERTISEMENT
:
335 /* For reaction involving unicast neighbor discovery
336 * message destined to the proxied address, pass it to
346 * The proxying router can't forward traffic sent to a link-local
347 * address, so signal the sender and discard the packet. This
348 * behavior is clarified by the MIPv6 specification.
350 if (ipv6_addr_type(&hdr
->daddr
) & IPV6_ADDR_LINKLOCAL
) {
351 dst_link_failure(skb
);
358 static inline int ip6_forward_finish(struct net
*net
, struct sock
*sk
,
361 return dst_output(net
, sk
, skb
);
364 static unsigned int ip6_dst_mtu_forward(const struct dst_entry
*dst
)
367 struct inet6_dev
*idev
;
369 if (dst_metric_locked(dst
, RTAX_MTU
)) {
370 mtu
= dst_metric_raw(dst
, RTAX_MTU
);
377 idev
= __in6_dev_get(dst
->dev
);
379 mtu
= idev
->cnf
.mtu6
;
385 static bool ip6_pkt_too_big(const struct sk_buff
*skb
, unsigned int mtu
)
390 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */
391 if (IP6CB(skb
)->frag_max_size
&& IP6CB(skb
)->frag_max_size
> mtu
)
397 if (skb_is_gso(skb
) && skb_gso_validate_mtu(skb
, mtu
))
403 int ip6_forward(struct sk_buff
*skb
)
405 struct dst_entry
*dst
= skb_dst(skb
);
406 struct ipv6hdr
*hdr
= ipv6_hdr(skb
);
407 struct inet6_skb_parm
*opt
= IP6CB(skb
);
408 struct net
*net
= dev_net(dst
->dev
);
411 if (net
->ipv6
.devconf_all
->forwarding
== 0)
414 if (skb
->pkt_type
!= PACKET_HOST
)
417 if (unlikely(skb
->sk
))
420 if (skb_warn_if_lro(skb
))
423 if (!xfrm6_policy_check(NULL
, XFRM_POLICY_FWD
, skb
)) {
424 __IP6_INC_STATS(net
, ip6_dst_idev(dst
),
425 IPSTATS_MIB_INDISCARDS
);
429 skb_forward_csum(skb
);
432 * We DO NOT make any processing on
433 * RA packets, pushing them to user level AS IS
434 * without ane WARRANTY that application will be able
435 * to interpret them. The reason is that we
436 * cannot make anything clever here.
438 * We are not end-node, so that if packet contains
439 * AH/ESP, we cannot make anything.
440 * Defragmentation also would be mistake, RA packets
441 * cannot be fragmented, because there is no warranty
442 * that different fragments will go along one path. --ANK
444 if (unlikely(opt
->flags
& IP6SKB_ROUTERALERT
)) {
445 if (ip6_call_ra_chain(skb
, ntohs(opt
->ra
)))
450 * check and decrement ttl
452 if (hdr
->hop_limit
<= 1) {
453 /* Force OUTPUT device used as source address */
455 icmpv6_send(skb
, ICMPV6_TIME_EXCEED
, ICMPV6_EXC_HOPLIMIT
, 0);
456 __IP6_INC_STATS(net
, ip6_dst_idev(dst
),
457 IPSTATS_MIB_INHDRERRORS
);
463 /* XXX: idev->cnf.proxy_ndp? */
464 if (net
->ipv6
.devconf_all
->proxy_ndp
&&
465 pneigh_lookup(&nd_tbl
, net
, &hdr
->daddr
, skb
->dev
, 0)) {
466 int proxied
= ip6_forward_proxy_check(skb
);
468 return ip6_input(skb
);
469 else if (proxied
< 0) {
470 __IP6_INC_STATS(net
, ip6_dst_idev(dst
),
471 IPSTATS_MIB_INDISCARDS
);
476 if (!xfrm6_route_forward(skb
)) {
477 __IP6_INC_STATS(net
, ip6_dst_idev(dst
),
478 IPSTATS_MIB_INDISCARDS
);
483 /* IPv6 specs say nothing about it, but it is clear that we cannot
484 send redirects to source routed frames.
485 We don't send redirects to frames decapsulated from IPsec.
487 if (skb
->dev
== dst
->dev
&& opt
->srcrt
== 0 && !skb_sec_path(skb
)) {
488 struct in6_addr
*target
= NULL
;
489 struct inet_peer
*peer
;
493 * incoming and outgoing devices are the same
497 rt
= (struct rt6_info
*) dst
;
498 if (rt
->rt6i_flags
& RTF_GATEWAY
)
499 target
= &rt
->rt6i_gateway
;
501 target
= &hdr
->daddr
;
503 peer
= inet_getpeer_v6(net
->ipv6
.peers
, &hdr
->daddr
, 1);
505 /* Limit redirects both by destination (here)
506 and by source (inside ndisc_send_redirect)
508 if (inet_peer_xrlim_allow(peer
, 1*HZ
))
509 ndisc_send_redirect(skb
, target
);
513 int addrtype
= ipv6_addr_type(&hdr
->saddr
);
515 /* This check is security critical. */
516 if (addrtype
== IPV6_ADDR_ANY
||
517 addrtype
& (IPV6_ADDR_MULTICAST
| IPV6_ADDR_LOOPBACK
))
519 if (addrtype
& IPV6_ADDR_LINKLOCAL
) {
520 icmpv6_send(skb
, ICMPV6_DEST_UNREACH
,
521 ICMPV6_NOT_NEIGHBOUR
, 0);
526 mtu
= ip6_dst_mtu_forward(dst
);
527 if (mtu
< IPV6_MIN_MTU
)
530 if (ip6_pkt_too_big(skb
, mtu
)) {
531 /* Again, force OUTPUT device used as source address */
533 icmpv6_send(skb
, ICMPV6_PKT_TOOBIG
, 0, mtu
);
534 __IP6_INC_STATS(net
, ip6_dst_idev(dst
),
535 IPSTATS_MIB_INTOOBIGERRORS
);
536 __IP6_INC_STATS(net
, ip6_dst_idev(dst
),
537 IPSTATS_MIB_FRAGFAILS
);
542 if (skb_cow(skb
, dst
->dev
->hard_header_len
)) {
543 __IP6_INC_STATS(net
, ip6_dst_idev(dst
),
544 IPSTATS_MIB_OUTDISCARDS
);
550 /* Mangling hops number delayed to point after skb COW */
554 __IP6_INC_STATS(net
, ip6_dst_idev(dst
), IPSTATS_MIB_OUTFORWDATAGRAMS
);
555 __IP6_ADD_STATS(net
, ip6_dst_idev(dst
), IPSTATS_MIB_OUTOCTETS
, skb
->len
);
556 return NF_HOOK(NFPROTO_IPV6
, NF_INET_FORWARD
,
557 net
, NULL
, skb
, skb
->dev
, dst
->dev
,
561 __IP6_INC_STATS(net
, ip6_dst_idev(dst
), IPSTATS_MIB_INADDRERRORS
);
567 static void ip6_copy_metadata(struct sk_buff
*to
, struct sk_buff
*from
)
569 to
->pkt_type
= from
->pkt_type
;
570 to
->priority
= from
->priority
;
571 to
->protocol
= from
->protocol
;
573 skb_dst_set(to
, dst_clone(skb_dst(from
)));
575 to
->mark
= from
->mark
;
577 #ifdef CONFIG_NET_SCHED
578 to
->tc_index
= from
->tc_index
;
581 skb_copy_secmark(to
, from
);
584 int ip6_fragment(struct net
*net
, struct sock
*sk
, struct sk_buff
*skb
,
585 int (*output
)(struct net
*, struct sock
*, struct sk_buff
*))
587 struct sk_buff
*frag
;
588 struct rt6_info
*rt
= (struct rt6_info
*)skb_dst(skb
);
589 struct ipv6_pinfo
*np
= skb
->sk
&& !dev_recursion_level() ?
590 inet6_sk(skb
->sk
) : NULL
;
591 struct ipv6hdr
*tmp_hdr
;
593 unsigned int mtu
, hlen
, left
, len
;
596 int ptr
, offset
= 0, err
= 0;
597 u8
*prevhdr
, nexthdr
= 0;
599 err
= ip6_find_1stfragopt(skb
, &prevhdr
);
605 mtu
= ip6_skb_dst_mtu(skb
);
607 /* We must not fragment if the socket is set to force MTU discovery
608 * or if the skb it not generated by a local socket.
610 if (unlikely(!skb
->ignore_df
&& skb
->len
> mtu
))
613 if (IP6CB(skb
)->frag_max_size
) {
614 if (IP6CB(skb
)->frag_max_size
> mtu
)
617 /* don't send fragments larger than what we received */
618 mtu
= IP6CB(skb
)->frag_max_size
;
619 if (mtu
< IPV6_MIN_MTU
)
623 if (np
&& np
->frag_size
< mtu
) {
627 if (mtu
< hlen
+ sizeof(struct frag_hdr
) + 8)
629 mtu
-= hlen
+ sizeof(struct frag_hdr
);
631 frag_id
= ipv6_select_ident(net
, &ipv6_hdr(skb
)->daddr
,
632 &ipv6_hdr(skb
)->saddr
);
634 if (skb
->ip_summed
== CHECKSUM_PARTIAL
&&
635 (err
= skb_checksum_help(skb
)))
638 hroom
= LL_RESERVED_SPACE(rt
->dst
.dev
);
639 if (skb_has_frag_list(skb
)) {
640 unsigned int first_len
= skb_pagelen(skb
);
641 struct sk_buff
*frag2
;
643 if (first_len
- hlen
> mtu
||
644 ((first_len
- hlen
) & 7) ||
646 skb_headroom(skb
) < (hroom
+ sizeof(struct frag_hdr
)))
649 skb_walk_frags(skb
, frag
) {
650 /* Correct geometry. */
651 if (frag
->len
> mtu
||
652 ((frag
->len
& 7) && frag
->next
) ||
653 skb_headroom(frag
) < (hlen
+ hroom
+ sizeof(struct frag_hdr
)))
654 goto slow_path_clean
;
656 /* Partially cloned skb? */
657 if (skb_shared(frag
))
658 goto slow_path_clean
;
663 frag
->destructor
= sock_wfree
;
665 skb
->truesize
-= frag
->truesize
;
672 *prevhdr
= NEXTHDR_FRAGMENT
;
673 tmp_hdr
= kmemdup(skb_network_header(skb
), hlen
, GFP_ATOMIC
);
675 IP6_INC_STATS(net
, ip6_dst_idev(skb_dst(skb
)),
676 IPSTATS_MIB_FRAGFAILS
);
680 frag
= skb_shinfo(skb
)->frag_list
;
681 skb_frag_list_init(skb
);
683 __skb_pull(skb
, hlen
);
684 fh
= (struct frag_hdr
*)__skb_push(skb
, sizeof(struct frag_hdr
));
685 __skb_push(skb
, hlen
);
686 skb_reset_network_header(skb
);
687 memcpy(skb_network_header(skb
), tmp_hdr
, hlen
);
689 fh
->nexthdr
= nexthdr
;
691 fh
->frag_off
= htons(IP6_MF
);
692 fh
->identification
= frag_id
;
694 first_len
= skb_pagelen(skb
);
695 skb
->data_len
= first_len
- skb_headlen(skb
);
696 skb
->len
= first_len
;
697 ipv6_hdr(skb
)->payload_len
= htons(first_len
-
698 sizeof(struct ipv6hdr
));
703 /* Prepare header of the next frame,
704 * before previous one went down. */
706 frag
->ip_summed
= CHECKSUM_NONE
;
707 skb_reset_transport_header(frag
);
708 fh
= (struct frag_hdr
*)__skb_push(frag
, sizeof(struct frag_hdr
));
709 __skb_push(frag
, hlen
);
710 skb_reset_network_header(frag
);
711 memcpy(skb_network_header(frag
), tmp_hdr
,
713 offset
+= skb
->len
- hlen
- sizeof(struct frag_hdr
);
714 fh
->nexthdr
= nexthdr
;
716 fh
->frag_off
= htons(offset
);
718 fh
->frag_off
|= htons(IP6_MF
);
719 fh
->identification
= frag_id
;
720 ipv6_hdr(frag
)->payload_len
=
722 sizeof(struct ipv6hdr
));
723 ip6_copy_metadata(frag
, skb
);
726 err
= output(net
, sk
, skb
);
728 IP6_INC_STATS(net
, ip6_dst_idev(&rt
->dst
),
729 IPSTATS_MIB_FRAGCREATES
);
742 IP6_INC_STATS(net
, ip6_dst_idev(&rt
->dst
),
743 IPSTATS_MIB_FRAGOKS
);
748 kfree_skb_list(frag
);
750 IP6_INC_STATS(net
, ip6_dst_idev(&rt
->dst
),
751 IPSTATS_MIB_FRAGFAILS
);
756 skb_walk_frags(skb
, frag2
) {
760 frag2
->destructor
= NULL
;
761 skb
->truesize
+= frag2
->truesize
;
766 left
= skb
->len
- hlen
; /* Space per frame */
767 ptr
= hlen
; /* Where to start from */
770 * Fragment the datagram.
773 troom
= rt
->dst
.dev
->needed_tailroom
;
776 * Keep copying data until we run out.
779 u8
*fragnexthdr_offset
;
782 /* IF: it doesn't fit, use 'mtu' - the data space left */
785 /* IF: we are not sending up to and including the packet end
786 then align the next start on an eight byte boundary */
791 /* Allocate buffer */
792 frag
= alloc_skb(len
+ hlen
+ sizeof(struct frag_hdr
) +
793 hroom
+ troom
, GFP_ATOMIC
);
795 IP6_INC_STATS(net
, ip6_dst_idev(skb_dst(skb
)),
796 IPSTATS_MIB_FRAGFAILS
);
802 * Set up data on packet
805 ip6_copy_metadata(frag
, skb
);
806 skb_reserve(frag
, hroom
);
807 skb_put(frag
, len
+ hlen
+ sizeof(struct frag_hdr
));
808 skb_reset_network_header(frag
);
809 fh
= (struct frag_hdr
*)(skb_network_header(frag
) + hlen
);
810 frag
->transport_header
= (frag
->network_header
+ hlen
+
811 sizeof(struct frag_hdr
));
814 * Charge the memory for the fragment to any owner
818 skb_set_owner_w(frag
, skb
->sk
);
821 * Copy the packet header into the new buffer.
823 skb_copy_from_linear_data(skb
, skb_network_header(frag
), hlen
);
825 fragnexthdr_offset
= skb_network_header(frag
);
826 fragnexthdr_offset
+= prevhdr
- skb_network_header(skb
);
827 *fragnexthdr_offset
= NEXTHDR_FRAGMENT
;
830 * Build fragment header.
832 fh
->nexthdr
= nexthdr
;
834 fh
->identification
= frag_id
;
837 * Copy a block of the IP datagram.
839 BUG_ON(skb_copy_bits(skb
, ptr
, skb_transport_header(frag
),
843 fh
->frag_off
= htons(offset
);
845 fh
->frag_off
|= htons(IP6_MF
);
846 ipv6_hdr(frag
)->payload_len
= htons(frag
->len
-
847 sizeof(struct ipv6hdr
));
853 * Put this fragment into the sending queue.
855 err
= output(net
, sk
, frag
);
859 IP6_INC_STATS(net
, ip6_dst_idev(skb_dst(skb
)),
860 IPSTATS_MIB_FRAGCREATES
);
862 IP6_INC_STATS(net
, ip6_dst_idev(skb_dst(skb
)),
863 IPSTATS_MIB_FRAGOKS
);
868 if (skb
->sk
&& dst_allfrag(skb_dst(skb
)))
869 sk_nocaps_add(skb
->sk
, NETIF_F_GSO_MASK
);
871 skb
->dev
= skb_dst(skb
)->dev
;
872 icmpv6_send(skb
, ICMPV6_PKT_TOOBIG
, 0, mtu
);
876 IP6_INC_STATS(net
, ip6_dst_idev(skb_dst(skb
)),
877 IPSTATS_MIB_FRAGFAILS
);
882 static inline int ip6_rt_check(const struct rt6key
*rt_key
,
883 const struct in6_addr
*fl_addr
,
884 const struct in6_addr
*addr_cache
)
886 return (rt_key
->plen
!= 128 || !ipv6_addr_equal(fl_addr
, &rt_key
->addr
)) &&
887 (!addr_cache
|| !ipv6_addr_equal(fl_addr
, addr_cache
));
890 static struct dst_entry
*ip6_sk_dst_check(struct sock
*sk
,
891 struct dst_entry
*dst
,
892 const struct flowi6
*fl6
)
894 struct ipv6_pinfo
*np
= inet6_sk(sk
);
900 if (dst
->ops
->family
!= AF_INET6
) {
905 rt
= (struct rt6_info
*)dst
;
906 /* Yes, checking route validity in not connected
907 * case is not very simple. Take into account,
908 * that we do not support routing by source, TOS,
909 * and MSG_DONTROUTE --ANK (980726)
911 * 1. ip6_rt_check(): If route was host route,
912 * check that cached destination is current.
913 * If it is network route, we still may
914 * check its validity using saved pointer
915 * to the last used address: daddr_cache.
916 * We do not want to save whole address now,
917 * (because main consumer of this service
918 * is tcp, which has not this problem),
919 * so that the last trick works only on connected
921 * 2. oif also should be the same.
923 if (ip6_rt_check(&rt
->rt6i_dst
, &fl6
->daddr
, np
->daddr_cache
) ||
924 #ifdef CONFIG_IPV6_SUBTREES
925 ip6_rt_check(&rt
->rt6i_src
, &fl6
->saddr
, np
->saddr_cache
) ||
927 (!(fl6
->flowi6_flags
& FLOWI_FLAG_SKIP_NH_OIF
) &&
928 (fl6
->flowi6_oif
&& fl6
->flowi6_oif
!= dst
->dev
->ifindex
))) {
937 static int ip6_dst_lookup_tail(struct net
*net
, const struct sock
*sk
,
938 struct dst_entry
**dst
, struct flowi6
*fl6
)
940 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
947 /* The correct way to handle this would be to do
948 * ip6_route_get_saddr, and then ip6_route_output; however,
949 * the route-specific preferred source forces the
950 * ip6_route_output call _before_ ip6_route_get_saddr.
952 * In source specific routing (no src=any default route),
953 * ip6_route_output will fail given src=any saddr, though, so
954 * that's why we try it again later.
956 if (ipv6_addr_any(&fl6
->saddr
) && (!*dst
|| !(*dst
)->error
)) {
958 bool had_dst
= *dst
!= NULL
;
961 *dst
= ip6_route_output(net
, sk
, fl6
);
962 rt
= (*dst
)->error
? NULL
: (struct rt6_info
*)*dst
;
963 err
= ip6_route_get_saddr(net
, rt
, &fl6
->daddr
,
964 sk
? inet6_sk(sk
)->srcprefs
: 0,
967 goto out_err_release
;
969 /* If we had an erroneous initial result, pretend it
970 * never existed and let the SA-enabled version take
973 if (!had_dst
&& (*dst
)->error
) {
979 flags
|= RT6_LOOKUP_F_IFACE
;
983 *dst
= ip6_route_output_flags(net
, sk
, fl6
, flags
);
987 goto out_err_release
;
989 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
991 * Here if the dst entry we've looked up
992 * has a neighbour entry that is in the INCOMPLETE
993 * state and the src address from the flow is
994 * marked as OPTIMISTIC, we release the found
995 * dst entry and replace it instead with the
996 * dst entry of the nexthop router
998 rt
= (struct rt6_info
*) *dst
;
1000 n
= __ipv6_neigh_lookup_noref(rt
->dst
.dev
,
1001 rt6_nexthop(rt
, &fl6
->daddr
));
1002 err
= n
&& !(n
->nud_state
& NUD_VALID
) ? -EINVAL
: 0;
1003 rcu_read_unlock_bh();
1006 struct inet6_ifaddr
*ifp
;
1007 struct flowi6 fl_gw6
;
1010 ifp
= ipv6_get_ifaddr(net
, &fl6
->saddr
,
1013 redirect
= (ifp
&& ifp
->flags
& IFA_F_OPTIMISTIC
);
1019 * We need to get the dst entry for the
1020 * default router instead
1023 memcpy(&fl_gw6
, fl6
, sizeof(struct flowi6
));
1024 memset(&fl_gw6
.daddr
, 0, sizeof(struct in6_addr
));
1025 *dst
= ip6_route_output(net
, sk
, &fl_gw6
);
1026 err
= (*dst
)->error
;
1028 goto out_err_release
;
1032 if (ipv6_addr_v4mapped(&fl6
->saddr
) &&
1033 !(ipv6_addr_v4mapped(&fl6
->daddr
) || ipv6_addr_any(&fl6
->daddr
))) {
1034 err
= -EAFNOSUPPORT
;
1035 goto out_err_release
;
1044 if (err
== -ENETUNREACH
)
1045 IP6_INC_STATS(net
, NULL
, IPSTATS_MIB_OUTNOROUTES
);
1050 * ip6_dst_lookup - perform route lookup on flow
1051 * @sk: socket which provides route info
1052 * @dst: pointer to dst_entry * for result
1053 * @fl6: flow to lookup
1055 * This function performs a route lookup on the given flow.
1057 * It returns zero on success, or a standard errno code on error.
1059 int ip6_dst_lookup(struct net
*net
, struct sock
*sk
, struct dst_entry
**dst
,
1063 return ip6_dst_lookup_tail(net
, sk
, dst
, fl6
);
1065 EXPORT_SYMBOL_GPL(ip6_dst_lookup
);
1068 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec
1069 * @sk: socket which provides route info
1070 * @fl6: flow to lookup
1071 * @final_dst: final destination address for ipsec lookup
1073 * This function performs a route lookup on the given flow.
1075 * It returns a valid dst pointer on success, or a pointer encoded
1078 struct dst_entry
*ip6_dst_lookup_flow(const struct sock
*sk
, struct flowi6
*fl6
,
1079 const struct in6_addr
*final_dst
)
1081 struct dst_entry
*dst
= NULL
;
1084 err
= ip6_dst_lookup_tail(sock_net(sk
), sk
, &dst
, fl6
);
1086 return ERR_PTR(err
);
1088 fl6
->daddr
= *final_dst
;
1090 return xfrm_lookup_route(sock_net(sk
), dst
, flowi6_to_flowi(fl6
), sk
, 0);
1092 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow
);
1095 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1096 * @sk: socket which provides the dst cache and route info
1097 * @fl6: flow to lookup
1098 * @final_dst: final destination address for ipsec lookup
1100 * This function performs a route lookup on the given flow with the
1101 * possibility of using the cached route in the socket if it is valid.
1102 * It will take the socket dst lock when operating on the dst cache.
1103 * As a result, this function can only be used in process context.
1105 * It returns a valid dst pointer on success, or a pointer encoded
1108 struct dst_entry
*ip6_sk_dst_lookup_flow(struct sock
*sk
, struct flowi6
*fl6
,
1109 const struct in6_addr
*final_dst
)
1111 struct dst_entry
*dst
= sk_dst_check(sk
, inet6_sk(sk
)->dst_cookie
);
1113 dst
= ip6_sk_dst_check(sk
, dst
, fl6
);
1115 dst
= ip6_dst_lookup_flow(sk
, fl6
, final_dst
);
1119 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow
);
1121 static inline int ip6_ufo_append_data(struct sock
*sk
,
1122 struct sk_buff_head
*queue
,
1123 int getfrag(void *from
, char *to
, int offset
, int len
,
1124 int odd
, struct sk_buff
*skb
),
1125 void *from
, int length
, int hh_len
, int fragheaderlen
,
1126 int exthdrlen
, int transhdrlen
, int mtu
,
1127 unsigned int flags
, const struct flowi6
*fl6
)
1130 struct sk_buff
*skb
;
1133 /* There is support for UDP large send offload by network
1134 * device, so create one single skb packet containing complete
1137 skb
= skb_peek_tail(queue
);
1139 skb
= sock_alloc_send_skb(sk
,
1140 hh_len
+ fragheaderlen
+ transhdrlen
+ 20,
1141 (flags
& MSG_DONTWAIT
), &err
);
1145 /* reserve space for Hardware header */
1146 skb_reserve(skb
, hh_len
);
1148 /* create space for UDP/IP header */
1149 skb_put(skb
, fragheaderlen
+ transhdrlen
);
1151 /* initialize network header pointer */
1152 skb_set_network_header(skb
, exthdrlen
);
1154 /* initialize protocol header pointer */
1155 skb
->transport_header
= skb
->network_header
+ fragheaderlen
;
1157 skb
->protocol
= htons(ETH_P_IPV6
);
1160 __skb_queue_tail(queue
, skb
);
1161 } else if (skb_is_gso(skb
)) {
1165 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1166 /* Specify the length of each IPv6 datagram fragment.
1167 * It has to be a multiple of 8.
1169 skb_shinfo(skb
)->gso_size
= (mtu
- fragheaderlen
-
1170 sizeof(struct frag_hdr
)) & ~7;
1171 skb_shinfo(skb
)->gso_type
= SKB_GSO_UDP
;
1172 skb_shinfo(skb
)->ip6_frag_id
= ipv6_select_ident(sock_net(sk
),
1177 return skb_append_datato_frags(sk
, skb
, getfrag
, from
,
1178 (length
- transhdrlen
));
1181 static inline struct ipv6_opt_hdr
*ip6_opt_dup(struct ipv6_opt_hdr
*src
,
1184 return src
? kmemdup(src
, (src
->hdrlen
+ 1) * 8, gfp
) : NULL
;
1187 static inline struct ipv6_rt_hdr
*ip6_rthdr_dup(struct ipv6_rt_hdr
*src
,
1190 return src
? kmemdup(src
, (src
->hdrlen
+ 1) * 8, gfp
) : NULL
;
1193 static void ip6_append_data_mtu(unsigned int *mtu
,
1195 unsigned int fragheaderlen
,
1196 struct sk_buff
*skb
,
1197 struct rt6_info
*rt
,
1198 unsigned int orig_mtu
)
1200 if (!(rt
->dst
.flags
& DST_XFRM_TUNNEL
)) {
1202 /* first fragment, reserve header_len */
1203 *mtu
= orig_mtu
- rt
->dst
.header_len
;
1207 * this fragment is not first, the headers
1208 * space is regarded as data space.
1212 *maxfraglen
= ((*mtu
- fragheaderlen
) & ~7)
1213 + fragheaderlen
- sizeof(struct frag_hdr
);
1217 static int ip6_setup_cork(struct sock
*sk
, struct inet_cork_full
*cork
,
1218 struct inet6_cork
*v6_cork
, struct ipcm6_cookie
*ipc6
,
1219 struct rt6_info
*rt
, struct flowi6
*fl6
)
1221 struct ipv6_pinfo
*np
= inet6_sk(sk
);
1223 struct ipv6_txoptions
*opt
= ipc6
->opt
;
1229 if (WARN_ON(v6_cork
->opt
))
1232 v6_cork
->opt
= kzalloc(opt
->tot_len
, sk
->sk_allocation
);
1233 if (unlikely(!v6_cork
->opt
))
1236 v6_cork
->opt
->tot_len
= opt
->tot_len
;
1237 v6_cork
->opt
->opt_flen
= opt
->opt_flen
;
1238 v6_cork
->opt
->opt_nflen
= opt
->opt_nflen
;
1240 v6_cork
->opt
->dst0opt
= ip6_opt_dup(opt
->dst0opt
,
1242 if (opt
->dst0opt
&& !v6_cork
->opt
->dst0opt
)
1245 v6_cork
->opt
->dst1opt
= ip6_opt_dup(opt
->dst1opt
,
1247 if (opt
->dst1opt
&& !v6_cork
->opt
->dst1opt
)
1250 v6_cork
->opt
->hopopt
= ip6_opt_dup(opt
->hopopt
,
1252 if (opt
->hopopt
&& !v6_cork
->opt
->hopopt
)
1255 v6_cork
->opt
->srcrt
= ip6_rthdr_dup(opt
->srcrt
,
1257 if (opt
->srcrt
&& !v6_cork
->opt
->srcrt
)
1260 /* need source address above miyazawa*/
1263 cork
->base
.dst
= &rt
->dst
;
1264 cork
->fl
.u
.ip6
= *fl6
;
1265 v6_cork
->hop_limit
= ipc6
->hlimit
;
1266 v6_cork
->tclass
= ipc6
->tclass
;
1267 if (rt
->dst
.flags
& DST_XFRM_TUNNEL
)
1268 mtu
= np
->pmtudisc
>= IPV6_PMTUDISC_PROBE
?
1269 rt
->dst
.dev
->mtu
: dst_mtu(&rt
->dst
);
1271 mtu
= np
->pmtudisc
>= IPV6_PMTUDISC_PROBE
?
1272 rt
->dst
.dev
->mtu
: dst_mtu(rt
->dst
.path
);
1273 if (np
->frag_size
< mtu
) {
1275 mtu
= np
->frag_size
;
1277 cork
->base
.fragsize
= mtu
;
1278 if (dst_allfrag(rt
->dst
.path
))
1279 cork
->base
.flags
|= IPCORK_ALLFRAG
;
1280 cork
->base
.length
= 0;
1285 static int __ip6_append_data(struct sock
*sk
,
1287 struct sk_buff_head
*queue
,
1288 struct inet_cork
*cork
,
1289 struct inet6_cork
*v6_cork
,
1290 struct page_frag
*pfrag
,
1291 int getfrag(void *from
, char *to
, int offset
,
1292 int len
, int odd
, struct sk_buff
*skb
),
1293 void *from
, int length
, int transhdrlen
,
1294 unsigned int flags
, struct ipcm6_cookie
*ipc6
,
1295 const struct sockcm_cookie
*sockc
)
1297 struct sk_buff
*skb
, *skb_prev
= NULL
;
1298 unsigned int maxfraglen
, fragheaderlen
, mtu
, orig_mtu
;
1300 int dst_exthdrlen
= 0;
1307 struct rt6_info
*rt
= (struct rt6_info
*)cork
->dst
;
1308 struct ipv6_txoptions
*opt
= v6_cork
->opt
;
1309 int csummode
= CHECKSUM_NONE
;
1310 unsigned int maxnonfragsize
, headersize
;
1312 skb
= skb_peek_tail(queue
);
1314 exthdrlen
= opt
? opt
->opt_flen
: 0;
1315 dst_exthdrlen
= rt
->dst
.header_len
- rt
->rt6i_nfheader_len
;
1318 mtu
= cork
->fragsize
;
1321 hh_len
= LL_RESERVED_SPACE(rt
->dst
.dev
);
1323 fragheaderlen
= sizeof(struct ipv6hdr
) + rt
->rt6i_nfheader_len
+
1324 (opt
? opt
->opt_nflen
: 0);
1325 maxfraglen
= ((mtu
- fragheaderlen
) & ~7) + fragheaderlen
-
1326 sizeof(struct frag_hdr
);
1328 headersize
= sizeof(struct ipv6hdr
) +
1329 (opt
? opt
->opt_flen
+ opt
->opt_nflen
: 0) +
1330 (dst_allfrag(&rt
->dst
) ?
1331 sizeof(struct frag_hdr
) : 0) +
1332 rt
->rt6i_nfheader_len
;
1334 if (cork
->length
+ length
> mtu
- headersize
&& ipc6
->dontfrag
&&
1335 (sk
->sk_protocol
== IPPROTO_UDP
||
1336 sk
->sk_protocol
== IPPROTO_RAW
)) {
1337 ipv6_local_rxpmtu(sk
, fl6
, mtu
- headersize
+
1338 sizeof(struct ipv6hdr
));
1342 if (ip6_sk_ignore_df(sk
))
1343 maxnonfragsize
= sizeof(struct ipv6hdr
) + IPV6_MAXPLEN
;
1345 maxnonfragsize
= mtu
;
1347 if (cork
->length
+ length
> maxnonfragsize
- headersize
) {
1349 ipv6_local_error(sk
, EMSGSIZE
, fl6
,
1351 sizeof(struct ipv6hdr
));
1355 /* CHECKSUM_PARTIAL only with no extension headers and when
1356 * we are not going to fragment
1358 if (transhdrlen
&& sk
->sk_protocol
== IPPROTO_UDP
&&
1359 headersize
== sizeof(struct ipv6hdr
) &&
1360 length
<= mtu
- headersize
&&
1361 !(flags
& MSG_MORE
) &&
1362 rt
->dst
.dev
->features
& (NETIF_F_IPV6_CSUM
| NETIF_F_HW_CSUM
))
1363 csummode
= CHECKSUM_PARTIAL
;
1365 if (sk
->sk_type
== SOCK_DGRAM
|| sk
->sk_type
== SOCK_RAW
) {
1366 sock_tx_timestamp(sk
, sockc
->tsflags
, &tx_flags
);
1367 if (tx_flags
& SKBTX_ANY_SW_TSTAMP
&&
1368 sk
->sk_tsflags
& SOF_TIMESTAMPING_OPT_ID
)
1369 tskey
= sk
->sk_tskey
++;
1373 * Let's try using as much space as possible.
1374 * Use MTU if total length of the message fits into the MTU.
1375 * Otherwise, we need to reserve fragment header and
1376 * fragment alignment (= 8-15 octects, in total).
1378 * Note that we may need to "move" the data from the tail of
1379 * of the buffer to the new fragment when we split
1382 * FIXME: It may be fragmented into multiple chunks
1383 * at once if non-fragmentable extension headers
1388 cork
->length
+= length
;
1389 if ((((length
+ fragheaderlen
) > mtu
) ||
1390 (skb
&& skb_is_gso(skb
))) &&
1391 (sk
->sk_protocol
== IPPROTO_UDP
) &&
1392 (rt
->dst
.dev
->features
& NETIF_F_UFO
) && !rt
->dst
.header_len
&&
1393 (sk
->sk_type
== SOCK_DGRAM
) && !udp_get_no_check6_tx(sk
)) {
1394 err
= ip6_ufo_append_data(sk
, queue
, getfrag
, from
, length
,
1395 hh_len
, fragheaderlen
, exthdrlen
,
1396 transhdrlen
, mtu
, flags
, fl6
);
1405 while (length
> 0) {
1406 /* Check if the remaining data fits into current packet. */
1407 copy
= (cork
->length
<= mtu
&& !(cork
->flags
& IPCORK_ALLFRAG
) ? mtu
: maxfraglen
) - skb
->len
;
1409 copy
= maxfraglen
- skb
->len
;
1413 unsigned int datalen
;
1414 unsigned int fraglen
;
1415 unsigned int fraggap
;
1416 unsigned int alloclen
;
1418 /* There's no room in the current skb */
1420 fraggap
= skb
->len
- maxfraglen
;
1423 /* update mtu and maxfraglen if necessary */
1424 if (!skb
|| !skb_prev
)
1425 ip6_append_data_mtu(&mtu
, &maxfraglen
,
1426 fragheaderlen
, skb
, rt
,
1432 * If remaining data exceeds the mtu,
1433 * we know we need more fragment(s).
1435 datalen
= length
+ fraggap
;
1437 if (datalen
> (cork
->length
<= mtu
&& !(cork
->flags
& IPCORK_ALLFRAG
) ? mtu
: maxfraglen
) - fragheaderlen
)
1438 datalen
= maxfraglen
- fragheaderlen
- rt
->dst
.trailer_len
;
1439 if ((flags
& MSG_MORE
) &&
1440 !(rt
->dst
.dev
->features
&NETIF_F_SG
))
1443 alloclen
= datalen
+ fragheaderlen
;
1445 alloclen
+= dst_exthdrlen
;
1447 if (datalen
!= length
+ fraggap
) {
1449 * this is not the last fragment, the trailer
1450 * space is regarded as data space.
1452 datalen
+= rt
->dst
.trailer_len
;
1455 alloclen
+= rt
->dst
.trailer_len
;
1456 fraglen
= datalen
+ fragheaderlen
;
1459 * We just reserve space for fragment header.
1460 * Note: this may be overallocation if the message
1461 * (without MSG_MORE) fits into the MTU.
1463 alloclen
+= sizeof(struct frag_hdr
);
1465 copy
= datalen
- transhdrlen
- fraggap
;
1471 skb
= sock_alloc_send_skb(sk
,
1473 (flags
& MSG_DONTWAIT
), &err
);
1476 if (atomic_read(&sk
->sk_wmem_alloc
) <=
1478 skb
= sock_wmalloc(sk
,
1479 alloclen
+ hh_len
, 1,
1487 * Fill in the control structures
1489 skb
->protocol
= htons(ETH_P_IPV6
);
1490 skb
->ip_summed
= csummode
;
1492 /* reserve for fragmentation and ipsec header */
1493 skb_reserve(skb
, hh_len
+ sizeof(struct frag_hdr
) +
1496 /* Only the initial fragment is time stamped */
1497 skb_shinfo(skb
)->tx_flags
= tx_flags
;
1499 skb_shinfo(skb
)->tskey
= tskey
;
1503 * Find where to start putting bytes
1505 data
= skb_put(skb
, fraglen
);
1506 skb_set_network_header(skb
, exthdrlen
);
1507 data
+= fragheaderlen
;
1508 skb
->transport_header
= (skb
->network_header
+
1511 skb
->csum
= skb_copy_and_csum_bits(
1512 skb_prev
, maxfraglen
,
1513 data
+ transhdrlen
, fraggap
, 0);
1514 skb_prev
->csum
= csum_sub(skb_prev
->csum
,
1517 pskb_trim_unique(skb_prev
, maxfraglen
);
1520 getfrag(from
, data
+ transhdrlen
, offset
,
1521 copy
, fraggap
, skb
) < 0) {
1528 length
-= datalen
- fraggap
;
1534 * Put the packet on the pending queue
1536 __skb_queue_tail(queue
, skb
);
1543 if (!(rt
->dst
.dev
->features
&NETIF_F_SG
)) {
1547 if (getfrag(from
, skb_put(skb
, copy
),
1548 offset
, copy
, off
, skb
) < 0) {
1549 __skb_trim(skb
, off
);
1554 int i
= skb_shinfo(skb
)->nr_frags
;
1557 if (!sk_page_frag_refill(sk
, pfrag
))
1560 if (!skb_can_coalesce(skb
, i
, pfrag
->page
,
1563 if (i
== MAX_SKB_FRAGS
)
1566 __skb_fill_page_desc(skb
, i
, pfrag
->page
,
1568 skb_shinfo(skb
)->nr_frags
= ++i
;
1569 get_page(pfrag
->page
);
1571 copy
= min_t(int, copy
, pfrag
->size
- pfrag
->offset
);
1573 page_address(pfrag
->page
) + pfrag
->offset
,
1574 offset
, copy
, skb
->len
, skb
) < 0)
1577 pfrag
->offset
+= copy
;
1578 skb_frag_size_add(&skb_shinfo(skb
)->frags
[i
- 1], copy
);
1580 skb
->data_len
+= copy
;
1581 skb
->truesize
+= copy
;
1582 atomic_add(copy
, &sk
->sk_wmem_alloc
);
1593 cork
->length
-= length
;
1594 IP6_INC_STATS(sock_net(sk
), rt
->rt6i_idev
, IPSTATS_MIB_OUTDISCARDS
);
1598 int ip6_append_data(struct sock
*sk
,
1599 int getfrag(void *from
, char *to
, int offset
, int len
,
1600 int odd
, struct sk_buff
*skb
),
1601 void *from
, int length
, int transhdrlen
,
1602 struct ipcm6_cookie
*ipc6
, struct flowi6
*fl6
,
1603 struct rt6_info
*rt
, unsigned int flags
,
1604 const struct sockcm_cookie
*sockc
)
1606 struct inet_sock
*inet
= inet_sk(sk
);
1607 struct ipv6_pinfo
*np
= inet6_sk(sk
);
1611 if (flags
&MSG_PROBE
)
1613 if (skb_queue_empty(&sk
->sk_write_queue
)) {
1617 err
= ip6_setup_cork(sk
, &inet
->cork
, &np
->cork
,
1622 exthdrlen
= (ipc6
->opt
? ipc6
->opt
->opt_flen
: 0);
1623 length
+= exthdrlen
;
1624 transhdrlen
+= exthdrlen
;
1626 fl6
= &inet
->cork
.fl
.u
.ip6
;
1630 return __ip6_append_data(sk
, fl6
, &sk
->sk_write_queue
, &inet
->cork
.base
,
1631 &np
->cork
, sk_page_frag(sk
), getfrag
,
1632 from
, length
, transhdrlen
, flags
, ipc6
, sockc
);
1634 EXPORT_SYMBOL_GPL(ip6_append_data
);
1636 static void ip6_cork_release(struct inet_cork_full
*cork
,
1637 struct inet6_cork
*v6_cork
)
1640 kfree(v6_cork
->opt
->dst0opt
);
1641 kfree(v6_cork
->opt
->dst1opt
);
1642 kfree(v6_cork
->opt
->hopopt
);
1643 kfree(v6_cork
->opt
->srcrt
);
1644 kfree(v6_cork
->opt
);
1645 v6_cork
->opt
= NULL
;
1648 if (cork
->base
.dst
) {
1649 dst_release(cork
->base
.dst
);
1650 cork
->base
.dst
= NULL
;
1651 cork
->base
.flags
&= ~IPCORK_ALLFRAG
;
1653 memset(&cork
->fl
, 0, sizeof(cork
->fl
));
1656 struct sk_buff
*__ip6_make_skb(struct sock
*sk
,
1657 struct sk_buff_head
*queue
,
1658 struct inet_cork_full
*cork
,
1659 struct inet6_cork
*v6_cork
)
1661 struct sk_buff
*skb
, *tmp_skb
;
1662 struct sk_buff
**tail_skb
;
1663 struct in6_addr final_dst_buf
, *final_dst
= &final_dst_buf
;
1664 struct ipv6_pinfo
*np
= inet6_sk(sk
);
1665 struct net
*net
= sock_net(sk
);
1666 struct ipv6hdr
*hdr
;
1667 struct ipv6_txoptions
*opt
= v6_cork
->opt
;
1668 struct rt6_info
*rt
= (struct rt6_info
*)cork
->base
.dst
;
1669 struct flowi6
*fl6
= &cork
->fl
.u
.ip6
;
1670 unsigned char proto
= fl6
->flowi6_proto
;
1672 skb
= __skb_dequeue(queue
);
1675 tail_skb
= &(skb_shinfo(skb
)->frag_list
);
1677 /* move skb->data to ip header from ext header */
1678 if (skb
->data
< skb_network_header(skb
))
1679 __skb_pull(skb
, skb_network_offset(skb
));
1680 while ((tmp_skb
= __skb_dequeue(queue
)) != NULL
) {
1681 __skb_pull(tmp_skb
, skb_network_header_len(skb
));
1682 *tail_skb
= tmp_skb
;
1683 tail_skb
= &(tmp_skb
->next
);
1684 skb
->len
+= tmp_skb
->len
;
1685 skb
->data_len
+= tmp_skb
->len
;
1686 skb
->truesize
+= tmp_skb
->truesize
;
1687 tmp_skb
->destructor
= NULL
;
1691 /* Allow local fragmentation. */
1692 skb
->ignore_df
= ip6_sk_ignore_df(sk
);
1694 *final_dst
= fl6
->daddr
;
1695 __skb_pull(skb
, skb_network_header_len(skb
));
1696 if (opt
&& opt
->opt_flen
)
1697 ipv6_push_frag_opts(skb
, opt
, &proto
);
1698 if (opt
&& opt
->opt_nflen
)
1699 ipv6_push_nfrag_opts(skb
, opt
, &proto
, &final_dst
, &fl6
->saddr
);
1701 skb_push(skb
, sizeof(struct ipv6hdr
));
1702 skb_reset_network_header(skb
);
1703 hdr
= ipv6_hdr(skb
);
1705 ip6_flow_hdr(hdr
, v6_cork
->tclass
,
1706 ip6_make_flowlabel(net
, skb
, fl6
->flowlabel
,
1707 np
->autoflowlabel
, fl6
));
1708 hdr
->hop_limit
= v6_cork
->hop_limit
;
1709 hdr
->nexthdr
= proto
;
1710 hdr
->saddr
= fl6
->saddr
;
1711 hdr
->daddr
= *final_dst
;
1713 skb
->priority
= sk
->sk_priority
;
1714 skb
->mark
= sk
->sk_mark
;
1716 skb_dst_set(skb
, dst_clone(&rt
->dst
));
1717 IP6_UPD_PO_STATS(net
, rt
->rt6i_idev
, IPSTATS_MIB_OUT
, skb
->len
);
1718 if (proto
== IPPROTO_ICMPV6
) {
1719 struct inet6_dev
*idev
= ip6_dst_idev(skb_dst(skb
));
1721 ICMP6MSGOUT_INC_STATS(net
, idev
, icmp6_hdr(skb
)->icmp6_type
);
1722 ICMP6_INC_STATS(net
, idev
, ICMP6_MIB_OUTMSGS
);
1725 ip6_cork_release(cork
, v6_cork
);
1730 int ip6_send_skb(struct sk_buff
*skb
)
1732 struct net
*net
= sock_net(skb
->sk
);
1733 struct rt6_info
*rt
= (struct rt6_info
*)skb_dst(skb
);
1736 err
= ip6_local_out(net
, skb
->sk
, skb
);
1739 err
= net_xmit_errno(err
);
1741 IP6_INC_STATS(net
, rt
->rt6i_idev
,
1742 IPSTATS_MIB_OUTDISCARDS
);
1748 int ip6_push_pending_frames(struct sock
*sk
)
1750 struct sk_buff
*skb
;
1752 skb
= ip6_finish_skb(sk
);
1756 return ip6_send_skb(skb
);
1758 EXPORT_SYMBOL_GPL(ip6_push_pending_frames
);
1760 static void __ip6_flush_pending_frames(struct sock
*sk
,
1761 struct sk_buff_head
*queue
,
1762 struct inet_cork_full
*cork
,
1763 struct inet6_cork
*v6_cork
)
1765 struct sk_buff
*skb
;
1767 while ((skb
= __skb_dequeue_tail(queue
)) != NULL
) {
1769 IP6_INC_STATS(sock_net(sk
), ip6_dst_idev(skb_dst(skb
)),
1770 IPSTATS_MIB_OUTDISCARDS
);
1774 ip6_cork_release(cork
, v6_cork
);
1777 void ip6_flush_pending_frames(struct sock
*sk
)
1779 __ip6_flush_pending_frames(sk
, &sk
->sk_write_queue
,
1780 &inet_sk(sk
)->cork
, &inet6_sk(sk
)->cork
);
1782 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames
);
1784 struct sk_buff
*ip6_make_skb(struct sock
*sk
,
1785 int getfrag(void *from
, char *to
, int offset
,
1786 int len
, int odd
, struct sk_buff
*skb
),
1787 void *from
, int length
, int transhdrlen
,
1788 struct ipcm6_cookie
*ipc6
, struct flowi6
*fl6
,
1789 struct rt6_info
*rt
, unsigned int flags
,
1790 const struct sockcm_cookie
*sockc
)
1792 struct inet_cork_full cork
;
1793 struct inet6_cork v6_cork
;
1794 struct sk_buff_head queue
;
1795 int exthdrlen
= (ipc6
->opt
? ipc6
->opt
->opt_flen
: 0);
1798 if (flags
& MSG_PROBE
)
1801 __skb_queue_head_init(&queue
);
1803 cork
.base
.flags
= 0;
1805 cork
.base
.opt
= NULL
;
1807 err
= ip6_setup_cork(sk
, &cork
, &v6_cork
, ipc6
, rt
, fl6
);
1809 return ERR_PTR(err
);
1811 if (ipc6
->dontfrag
< 0)
1812 ipc6
->dontfrag
= inet6_sk(sk
)->dontfrag
;
1814 err
= __ip6_append_data(sk
, fl6
, &queue
, &cork
.base
, &v6_cork
,
1815 ¤t
->task_frag
, getfrag
, from
,
1816 length
+ exthdrlen
, transhdrlen
+ exthdrlen
,
1817 flags
, ipc6
, sockc
);
1819 __ip6_flush_pending_frames(sk
, &queue
, &cork
, &v6_cork
);
1820 return ERR_PTR(err
);
1823 return __ip6_make_skb(sk
, &queue
, &cork
, &v6_cork
);