udp6: don't make extra copies of iflow

udpv6_sendmsg() first initialises an on-stack 88B struct flowi6 and then
copies it into cork, which is expensive. Avoid the copy in corkless case
by initialising on-stack cork->fl directly.

The main part is a couple of lines under !corkreq check. The rest
converts fl6 variable to be a pointer.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 44b7ca9bd78eaaa06485aade7d1b97efba290c2f..cfcf08c3df4ddc9ba3aad1a7ade6757dc8635d88 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -1294,7 +1294,8 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
        struct ipv6_txoptions *opt = NULL;
        struct ipv6_txoptions *opt_to_free = NULL;
        struct ip6_flowlabel *flowlabel = NULL;
-       struct flowi6 fl6;
+       struct inet_cork_full cork;
+       struct flowi6 *fl6 = &cork.fl.u.ip6;
        struct dst_entry *dst;
        struct ipcm6_cookie ipc6;
        int addr_len = msg->msg_namelen;
@@ -1384,19 +1385,19 @@ do_udp_sendmsg:
        }
        ulen += sizeof(struct udphdr);
 
-       memset(&fl6, 0, sizeof(fl6));
+       memset(fl6, 0, sizeof(*fl6));
 
        if (sin6) {
                if (sin6->sin6_port == 0)
                        return -EINVAL;
 
-               fl6.fl6_dport = sin6->sin6_port;
+               fl6->fl6_dport = sin6->sin6_port;
                daddr = &sin6->sin6_addr;
 
                if (np->sndflow) {
-                       fl6.flowlabel = sin6->sin6_flowinfo&IPV6_FLOWINFO_MASK;
-                       if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
-                               flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
+                       fl6->flowlabel = sin6->sin6_flowinfo&IPV6_FLOWINFO_MASK;
+                       if (fl6->flowlabel & IPV6_FLOWLABEL_MASK) {
+                               flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
                                if (IS_ERR(flowlabel))
                                        return -EINVAL;
                        }
@@ -1413,24 +1414,24 @@ do_udp_sendmsg:
                if (addr_len >= sizeof(struct sockaddr_in6) &&
                    sin6->sin6_scope_id &&
                    __ipv6_addr_needs_scope_id(__ipv6_addr_type(daddr)))
-                       fl6.flowi6_oif = sin6->sin6_scope_id;
+                       fl6->flowi6_oif = sin6->sin6_scope_id;
        } else {
                if (sk->sk_state != TCP_ESTABLISHED)
                        return -EDESTADDRREQ;
 
-               fl6.fl6_dport = inet->inet_dport;
+               fl6->fl6_dport = inet->inet_dport;
                daddr = &sk->sk_v6_daddr;
-               fl6.flowlabel = np->flow_label;
+               fl6->flowlabel = np->flow_label;
                connected = true;
        }
 
-       if (!fl6.flowi6_oif)
-               fl6.flowi6_oif = sk->sk_bound_dev_if;
+       if (!fl6->flowi6_oif)
+               fl6->flowi6_oif = sk->sk_bound_dev_if;
 
-       if (!fl6.flowi6_oif)
-               fl6.flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;
+       if (!fl6->flowi6_oif)
+               fl6->flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;
 
-       fl6.flowi6_uid = sk->sk_uid;
+       fl6->flowi6_uid = sk->sk_uid;
 
        if (msg->msg_controllen) {
                opt = &opt_space;
@@ -1440,14 +1441,14 @@ do_udp_sendmsg:
 
                err = udp_cmsg_send(sk, msg, &ipc6.gso_size);
                if (err > 0)
-                       err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, &fl6,
+                       err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, fl6,
                                                    &ipc6);
                if (err < 0) {
                        fl6_sock_release(flowlabel);
                        return err;
                }
-               if ((fl6.flowlabel&IPV6_FLOWLABEL_MASK) && !flowlabel) {
-                       flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
+               if ((fl6->flowlabel&IPV6_FLOWLABEL_MASK) && !flowlabel) {
+                       flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
                        if (IS_ERR(flowlabel))
                                return -EINVAL;
                }
@@ -1464,16 +1465,17 @@ do_udp_sendmsg:
        opt = ipv6_fixup_options(&opt_space, opt);
        ipc6.opt = opt;
 
-       fl6.flowi6_proto = sk->sk_protocol;
-       fl6.flowi6_mark = ipc6.sockc.mark;
-       fl6.daddr = *daddr;
-       if (ipv6_addr_any(&fl6.saddr) && !ipv6_addr_any(&np->saddr))
-               fl6.saddr = np->saddr;
-       fl6.fl6_sport = inet->inet_sport;
+       fl6->flowi6_proto = sk->sk_protocol;
+       fl6->flowi6_mark = ipc6.sockc.mark;
+       fl6->daddr = *daddr;
+       if (ipv6_addr_any(&fl6->saddr) && !ipv6_addr_any(&np->saddr))
+               fl6->saddr = np->saddr;
+       fl6->fl6_sport = inet->inet_sport;
 
        if (cgroup_bpf_enabled(CGROUP_UDP6_SENDMSG) && !connected) {
                err = BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk,
-                                          (struct sockaddr *)sin6, &fl6.saddr);
+                                          (struct sockaddr *)sin6,
+                                          &fl6->saddr);
                if (err)
                        goto out_no_dst;
                if (sin6) {
@@ -1489,32 +1491,32 @@ do_udp_sendmsg:
                                err = -EINVAL;
                                goto out_no_dst;
                        }
-                       fl6.fl6_dport = sin6->sin6_port;
-                       fl6.daddr = sin6->sin6_addr;
+                       fl6->fl6_dport = sin6->sin6_port;
+                       fl6->daddr = sin6->sin6_addr;
                }
        }
 
-       if (ipv6_addr_any(&fl6.daddr))
-               fl6.daddr.s6_addr[15] = 0x1; /* :: means loopback (BSD'ism) */
+       if (ipv6_addr_any(&fl6->daddr))
+               fl6->daddr.s6_addr[15] = 0x1; /* :: means loopback (BSD'ism) */
 
-       final_p = fl6_update_dst(&fl6, opt, &final);
+       final_p = fl6_update_dst(fl6, opt, &final);
        if (final_p)
                connected = false;
 
-       if (!fl6.flowi6_oif && ipv6_addr_is_multicast(&fl6.daddr)) {
-               fl6.flowi6_oif = np->mcast_oif;
+       if (!fl6->flowi6_oif && ipv6_addr_is_multicast(&fl6->daddr)) {
+               fl6->flowi6_oif = np->mcast_oif;
                connected = false;
-       } else if (!fl6.flowi6_oif)
-               fl6.flowi6_oif = np->ucast_oif;
+       } else if (!fl6->flowi6_oif)
+               fl6->flowi6_oif = np->ucast_oif;
 
-       security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
+       security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));
 
        if (ipc6.tclass < 0)
                ipc6.tclass = np->tclass;
 
-       fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);
+       fl6->flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6->flowlabel);
 
-       dst = ip6_sk_dst_lookup_flow(sk, &fl6, final_p, connected);
+       dst = ip6_sk_dst_lookup_flow(sk, fl6, final_p, connected);
        if (IS_ERR(dst)) {
                err = PTR_ERR(dst);
                dst = NULL;
@@ -1522,7 +1524,7 @@ do_udp_sendmsg:
        }
 
        if (ipc6.hlimit < 0)
-               ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
+               ipc6.hlimit = ip6_sk_dst_hoplimit(np, fl6, dst);
 
        if (msg->msg_flags&MSG_CONFIRM)
                goto do_confirm;
@@ -1530,18 +1532,15 @@ back_from_confirm:
 
        /* Lockless fast path for the non-corking case */
        if (!corkreq) {
-               struct inet_cork_full cork;
                struct sk_buff *skb;
 
-               cork.fl.u.ip6 = fl6;
-
                skb = ip6_make_skb(sk, getfrag, msg, ulen,
                                   sizeof(struct udphdr), &ipc6,
                                   (struct rt6_info *)dst,
                                   msg->msg_flags, &cork);
                err = PTR_ERR(skb);
                if (!IS_ERR_OR_NULL(skb))
-                       err = udp_v6_send_skb(skb, &fl6, &cork.base);
+                       err = udp_v6_send_skb(skb, fl6, &cork.base);
                goto out;
        }
 
@@ -1563,7 +1562,7 @@ do_append_data:
                ipc6.dontfrag = np->dontfrag;
        up->len += ulen;
        err = ip6_append_data(sk, getfrag, msg, ulen, sizeof(struct udphdr),
-                             &ipc6, &fl6, (struct rt6_info *)dst,
+                             &ipc6, fl6, (struct rt6_info *)dst,
                              corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
        if (err)
                udp_v6_flush_pending_frames(sk);
@@ -1598,7 +1597,7 @@ out_no_dst:
 
 do_confirm:
        if (msg->msg_flags & MSG_PROBE)
-               dst_confirm_neigh(dst, &fl6.daddr);
+               dst_confirm_neigh(dst, &fl6->daddr);
        if (!(msg->msg_flags&MSG_PROBE) || len)
                goto back_from_confirm;
        err = 0;