tcp: support externally provided ubufs

Teach tcp how to use external ubuf_info provided in msghdr and
also prepare it for managed frags by sprinkling
skb_zcopy_downgrade_managed() when it could mix managed and not managed
frags.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 028513d3e2a2bcf18022d9458e46c9a36c46eb2c..fdb80b9608fe9a8962a14fdaba729f12de411571 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1202,17 +1202,23 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
 
        flags = msg->msg_flags;
 
-       if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) {
+       if ((flags & MSG_ZEROCOPY) && size) {
                skb = tcp_write_queue_tail(sk);
-               uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb));
-               if (!uarg) {
-                       err = -ENOBUFS;
-                       goto out_err;
-               }
 
-               zc = sk->sk_route_caps & NETIF_F_SG;
-               if (!zc)
-                       uarg->zerocopy = 0;
+               if (msg->msg_ubuf) {
+                       uarg = msg->msg_ubuf;
+                       net_zcopy_get(uarg);
+                       zc = sk->sk_route_caps & NETIF_F_SG;
+               } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
+                       uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb));
+                       if (!uarg) {
+                               err = -ENOBUFS;
+                               goto out_err;
+                       }
+                       zc = sk->sk_route_caps & NETIF_F_SG;
+                       if (!zc)
+                               uarg->zerocopy = 0;
+               }
        }
 
        if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) &&
@@ -1335,8 +1341,13 @@ new_segment:
 
                        copy = min_t(int, copy, pfrag->size - pfrag->offset);
 
-                       if (tcp_downgrade_zcopy_pure(sk, skb) ||
-                           !sk_wmem_schedule(sk, copy))
+                       if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) {
+                               if (tcp_downgrade_zcopy_pure(sk, skb))
+                                       goto wait_for_space;
+                               skb_zcopy_downgrade_managed(skb);
+                       }
+
+                       if (!sk_wmem_schedule(sk, copy))
                                goto wait_for_space;
 
                        err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,