virtio-net: pack headroom into ctx for mergeable buffers

[mirror_ubuntu-bionic-kernel.git] / drivers / net / virtio_net.c

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index a871f45ecc79a438b2b43465d3719f240ff25cb5..3223a36ad9a81649a0552f538623012fd2c4b4bd 100644 (file)
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -270,6 +270,23 @@ static void skb_xmit_done(struct virtqueue *vq)
                netif_wake_subqueue(vi->dev, vq2txq(vq));
 }
 
+#define MRG_CTX_HEADER_SHIFT 22
+static void *mergeable_len_to_ctx(unsigned int truesize,
+                                 unsigned int headroom)
+{
+       return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
+}
+
+static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
+{
+       return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
+}
+
+static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
+{
+       return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
+}
+
 /* Called from bottom half context */
 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
                                   struct receive_queue *rq,
@@ -305,7 +322,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
        copy = len;
        if (copy > skb_tailroom(skb))
                copy = skb_tailroom(skb);
-       memcpy(skb_put(skb, copy), p, copy);
+       skb_put_data(skb, p, copy);
 
        len -= copy;
        offset += copy;
@@ -639,13 +656,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
        }
        rcu_read_unlock();
 
-       if (unlikely(len > (unsigned long)ctx)) {
+       truesize = mergeable_ctx_to_truesize(ctx);
+       if (unlikely(len > truesize)) {
                pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
                         dev->name, len, (unsigned long)ctx);
                dev->stats.rx_length_errors++;
                goto err_skb;
        }
-       truesize = (unsigned long)ctx;
+
        head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
        curr_skb = head_skb;
 
@@ -665,13 +683,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
                }
 
                page = virt_to_head_page(buf);
-               if (unlikely(len > (unsigned long)ctx)) {
+
+               truesize = mergeable_ctx_to_truesize(ctx);
+               if (unlikely(len > truesize)) {
                        pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
                                 dev->name, len, (unsigned long)ctx);
                        dev->stats.rx_length_errors++;
                        goto err_skb;
                }
-               truesize = (unsigned long)ctx;
 
                num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
                if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
@@ -889,7 +908,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
 
        buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
        buf += headroom; /* advance address leaving hole at front of pkt */
-       ctx = (void *)(unsigned long)len;
+       ctx = mergeable_len_to_ctx(len, headroom);
        get_page(alloc_frag->page);
        alloc_frag->offset += len + headroom;
        hole = alloc_frag->size - alloc_frag->offset;
@@ -1150,7 +1169,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
        struct virtio_net_hdr_mrg_rxbuf *hdr;
        const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
        struct virtnet_info *vi = sq->vq->vdev->priv;
-       unsigned num_sg;
+       int num_sg;
        unsigned hdr_len = vi->hdr_len;
        bool can_push;
 
@@ -1177,11 +1196,16 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
        if (can_push) {
                __skb_push(skb, hdr_len);
                num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
+               if (unlikely(num_sg < 0))
+                       return num_sg;
                /* Pull header back to avoid skew in tx bytes calculations. */
                __skb_pull(skb, hdr_len);
        } else {
                sg_set_buf(sq->sg, hdr, hdr_len);
-               num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
+               num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
+               if (unlikely(num_sg < 0))
+                       return num_sg;
+               num_sg++;
        }
        return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
 }
@@ -1797,6 +1821,7 @@ static void virtnet_freeze_down(struct virtio_device *vdev)
        flush_work(&vi->config_work);
 
        netif_device_detach(vi->dev);
+       netif_tx_disable(vi->dev);
        cancel_delayed_work_sync(&vi->refill);
 
        if (netif_running(vi->dev)) {
@@ -1950,16 +1975,18 @@ virtio_reset_err:
        return err;
 }
 
-static bool virtnet_xdp_query(struct net_device *dev)
+static u32 virtnet_xdp_query(struct net_device *dev)
 {
        struct virtnet_info *vi = netdev_priv(dev);
+       const struct bpf_prog *xdp_prog;
        int i;
 
        for (i = 0; i < vi->max_queue_pairs; i++) {
-               if (vi->rq[i].xdp_prog)
-                       return true;
+               xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog);
+               if (xdp_prog)
+                       return xdp_prog->aux->id;
        }
-       return false;
+       return 0;
 }
 
 static int virtnet_xdp(struct net_device *dev, struct netdev_xdp *xdp)
@@ -1968,7 +1995,8 @@ static int virtnet_xdp(struct net_device *dev, struct netdev_xdp *xdp)
        case XDP_SETUP_PROG:
                return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
        case XDP_QUERY_PROG:
-               xdp->prog_attached = virtnet_xdp_query(dev);
+               xdp->prog_id = virtnet_xdp_query(dev);
+               xdp->prog_attached = !!xdp->prog_id;
                return 0;
        default:
                return -EINVAL;
@@ -2220,6 +2248,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
        kfree(names);
        kfree(callbacks);
        kfree(vqs);
+       kfree(ctx);
 
        return 0;
 
@@ -2419,7 +2448,7 @@ static int virtnet_probe(struct virtio_device *vdev)
                        dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
 
                if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
-                       dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
+                       dev->hw_features |= NETIF_F_TSO
                                | NETIF_F_TSO_ECN | NETIF_F_TSO6;
                }
                /* Individual feature bits: what can host handle? */
@@ -2429,13 +2458,11 @@ static int virtnet_probe(struct virtio_device *vdev)
                        dev->hw_features |= NETIF_F_TSO6;
                if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
                        dev->hw_features |= NETIF_F_TSO_ECN;
-               if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
-                       dev->hw_features |= NETIF_F_UFO;
 
                dev->features |= NETIF_F_GSO_ROBUST;
 
                if (gso)
-                       dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
+                       dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
                /* (!csum && gso) case will be fixed by register_netdev() */
        }
        if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))