xen-netback: Handle guests with too many frags

Xen network protocol had implicit dependency on MAX_SKB_FRAGS. Netback has to
handle guests sending up to XEN_NETBK_LEGACY_SLOTS_MAX slots. To achieve that:
- create a new skb
- map the leftover slots to its frags (no linear buffer here!)
- chain it to the previous through skb_shinfo(skb)->frag_list
- map them
- copy and coalesce the frags into a brand new one and send it to the stack
- unmap the 2 old skb's pages

It's also introduces new stat counters, which help determine how often the guest
sends a packet with more than MAX_SKB_FRAGS frags.

NOTE: if bisect brought you here, you should apply the series up until
"xen-netback: Timeout packets in RX path", otherwise malicious guests can block
other guests by not releasing their sent packets.

Signed-off-by: Zoltan Kiss <zoltan.kiss@citrix.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
index 683d30160a7c016fea24c9185f580e0a52df35ac..f2f8a02afc36e72c2456f8675d92a2d048e76fe8 100644 (file)
--- a/drivers/net/xen-netback/common.h
+++ b/drivers/net/xen-netback/common.h
@@ -182,6 +182,7 @@ struct xenvif {
        unsigned long tx_zerocopy_sent;
        unsigned long tx_zerocopy_success;
        unsigned long tx_zerocopy_fail;
+       unsigned long tx_frag_overflow;
 
        /* Miscellaneous private stuff. */
        struct net_device *dev;

diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index 44df8581b4d7c363cd314e90431ed62a635e768f..b646039e539b8039080c54236ce40946c6c72a21 100644 (file)
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -253,6 +253,13 @@ static const struct xenvif_stat {
                "tx_zerocopy_fail",
                offsetof(struct xenvif, tx_zerocopy_fail)
        },
+       /* Number of packets exceeding MAX_SKB_FRAG slots. You should use
+        * a guest with the same MAX_SKB_FRAG
+        */
+       {
+               "tx_frag_overflow",
+               offsetof(struct xenvif, tx_frag_overflow)
+       },
 };
 
 static int xenvif_get_sset_count(struct net_device *dev, int string_set)

diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index 3cb586357df716999a91995aece35fbf0dfe19ab..58effc49f526b31fa914c704e547a800743f8c8f 100644 (file)
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -37,6 +37,7 @@
 #include <linux/kthread.h>
 #include <linux/if_vlan.h>
 #include <linux/udp.h>
+#include <linux/highmem.h>
 
 #include <net/tcp.h>
 
@@ -801,6 +802,23 @@ static inline void xenvif_tx_create_gop(struct xenvif *vif,
               sizeof(*txp));
 }
 
+static inline struct sk_buff *xenvif_alloc_skb(unsigned int size)
+{
+       struct sk_buff *skb =
+               alloc_skb(size + NET_SKB_PAD + NET_IP_ALIGN,
+                         GFP_ATOMIC | __GFP_NOWARN);
+       if (unlikely(skb == NULL))
+               return NULL;
+
+       /* Packets passed to netif_rx() must have some headroom. */
+       skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
+
+       /* Initialize it here to avoid later surprises */
+       skb_shinfo(skb)->destructor_arg = NULL;
+
+       return skb;
+}
+
 static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
                                                        struct sk_buff *skb,
                                                        struct xen_netif_tx_request *txp,
@@ -811,11 +829,16 @@ static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
        u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx;
        int start;
        pending_ring_idx_t index;
-       unsigned int nr_slots;
+       unsigned int nr_slots, frag_overflow = 0;
 
        /* At this point shinfo->nr_frags is in fact the number of
         * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
         */
+       if (shinfo->nr_frags > MAX_SKB_FRAGS) {
+               frag_overflow = shinfo->nr_frags - MAX_SKB_FRAGS;
+               BUG_ON(frag_overflow > MAX_SKB_FRAGS);
+               shinfo->nr_frags = MAX_SKB_FRAGS;
+       }
        nr_slots = shinfo->nr_frags;
 
        /* Skip first skb fragment if it is on same page as header fragment. */
@@ -829,7 +852,29 @@ static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
                frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx);
        }
 
-       BUG_ON(shinfo->nr_frags > MAX_SKB_FRAGS);
+       if (frag_overflow) {
+               struct sk_buff *nskb = xenvif_alloc_skb(0);
+               if (unlikely(nskb == NULL)) {
+                       if (net_ratelimit())
+                               netdev_err(vif->dev,
+                                          "Can't allocate the frag_list skb.\n");
+                       return NULL;
+               }
+
+               shinfo = skb_shinfo(nskb);
+               frags = shinfo->frags;
+
+               for (shinfo->nr_frags = 0; shinfo->nr_frags < frag_overflow;
+                    shinfo->nr_frags++, txp++, gop++) {
+                       index = pending_index(vif->pending_cons++);
+                       pending_idx = vif->pending_ring[index];
+                       xenvif_tx_create_gop(vif, pending_idx, txp, gop);
+                       frag_set_pending_idx(&frags[shinfo->nr_frags],
+                                            pending_idx);
+               }
+
+               skb_shinfo(skb)->frag_list = nskb;
+       }
 
        return gop;
 }
@@ -871,6 +916,7 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
        struct pending_tx_info *tx_info;
        int nr_frags = shinfo->nr_frags;
        int i, err, start;
+       struct sk_buff *first_skb = NULL;
 
        /* Check status of header. */
        err = gop->status;
@@ -882,6 +928,7 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
        /* Skip first skb fragment if it is on same page as header fragment. */
        start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
 
+check_frags:
        for (i = start; i < nr_frags; i++) {
                int j, newerr;
 
@@ -905,9 +952,11 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
                /* Not the first error? Preceding frags already invalidated. */
                if (err)
                        continue;
-
                /* First error: invalidate header and preceding fragments. */
-               pending_idx = XENVIF_TX_CB(skb)->pending_idx;
+               if (!first_skb)
+                       pending_idx = XENVIF_TX_CB(skb)->pending_idx;
+               else
+                       pending_idx = XENVIF_TX_CB(skb)->pending_idx;
                xenvif_idx_unmap(vif, pending_idx);
                for (j = start; j < i; j++) {
                        pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
@@ -918,6 +967,30 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
                err = newerr;
        }
 
+       if (skb_has_frag_list(skb)) {
+               first_skb = skb;
+               skb = shinfo->frag_list;
+               shinfo = skb_shinfo(skb);
+               nr_frags = shinfo->nr_frags;
+               start = 0;
+
+               goto check_frags;
+       }
+
+       /* There was a mapping error in the frag_list skb. We have to unmap
+        * the first skb's frags
+        */
+       if (first_skb && err) {
+               int j;
+               shinfo = skb_shinfo(first_skb);
+               pending_idx = XENVIF_TX_CB(skb)->pending_idx;
+               start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
+               for (j = start; j < shinfo->nr_frags; j++) {
+                       pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
+                       xenvif_idx_unmap(vif, pending_idx);
+               }
+       }
+
        *gopp = gop + 1;
        return err;
 }
@@ -1169,8 +1242,7 @@ static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
                            ret < XEN_NETBK_LEGACY_SLOTS_MAX) ?
                        PKT_PROT_LEN : txreq.size;
 
-               skb = alloc_skb(data_len + NET_SKB_PAD + NET_IP_ALIGN,
-                               GFP_ATOMIC | __GFP_NOWARN);
+               skb = xenvif_alloc_skb(data_len);
                if (unlikely(skb == NULL)) {
                        netdev_dbg(vif->dev,
                                   "Can't allocate a skb in start_xmit.\n");
@@ -1178,9 +1250,6 @@ static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
                        break;
                }
 
-               /* Packets passed to netif_rx() must have some headroom. */
-               skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
-
                if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
                        struct xen_netif_extra_info *gso;
                        gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
@@ -1231,6 +1300,71 @@ static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
        return gop - vif->tx_map_ops;
 }
 
+/* Consolidate skb with a frag_list into a brand new one with local pages on
+ * frags. Returns 0 or -ENOMEM if can't allocate new pages.
+ */
+static int xenvif_handle_frag_list(struct xenvif *vif, struct sk_buff *skb)
+{
+       unsigned int offset = skb_headlen(skb);
+       skb_frag_t frags[MAX_SKB_FRAGS];
+       int i;
+       struct ubuf_info *uarg;
+       struct sk_buff *nskb = skb_shinfo(skb)->frag_list;
+
+       vif->tx_zerocopy_sent += 2;
+       vif->tx_frag_overflow++;
+
+       xenvif_fill_frags(vif, nskb);
+       /* Subtract frags size, we will correct it later */
+       skb->truesize -= skb->data_len;
+       skb->len += nskb->len;
+       skb->data_len += nskb->len;
+
+       /* create a brand new frags array and coalesce there */
+       for (i = 0; offset < skb->len; i++) {
+               struct page *page;
+               unsigned int len;
+
+               BUG_ON(i >= MAX_SKB_FRAGS);
+               page = alloc_page(GFP_ATOMIC|__GFP_COLD);
+               if (!page) {
+                       int j;
+                       skb->truesize += skb->data_len;
+                       for (j = 0; j < i; j++)
+                               put_page(frags[j].page.p);
+                       return -ENOMEM;
+               }
+
+               if (offset + PAGE_SIZE < skb->len)
+                       len = PAGE_SIZE;
+               else
+                       len = skb->len - offset;
+               if (skb_copy_bits(skb, offset, page_address(page), len))
+                       BUG();
+
+               offset += len;
+               frags[i].page.p = page;
+               frags[i].page_offset = 0;
+               skb_frag_size_set(&frags[i], len);
+       }
+       /* swap out with old one */
+       memcpy(skb_shinfo(skb)->frags,
+              frags,
+              i * sizeof(skb_frag_t));
+       skb_shinfo(skb)->nr_frags = i;
+       skb->truesize += i * PAGE_SIZE;
+
+       /* remove traces of mapped pages and frag_list */
+       skb_frag_list_init(skb);
+       uarg = skb_shinfo(skb)->destructor_arg;
+       uarg->callback(uarg, true);
+       skb_shinfo(skb)->destructor_arg = NULL;
+
+       skb_shinfo(nskb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
+       kfree_skb(nskb);
+
+       return 0;
+}
 
 static int xenvif_tx_submit(struct xenvif *vif)
 {
@@ -1267,7 +1401,6 @@ static int xenvif_tx_submit(struct xenvif *vif)
                                &vif->pending_tx_info[pending_idx].callback_struct;
                } else {
                        /* Schedule a response immediately. */
-                       skb_shinfo(skb)->destructor_arg = NULL;
                        xenvif_idx_unmap(vif, pending_idx);
                }
 
@@ -1278,6 +1411,17 @@ static int xenvif_tx_submit(struct xenvif *vif)
 
                xenvif_fill_frags(vif, skb);
 
+               if (unlikely(skb_has_frag_list(skb))) {
+                       if (xenvif_handle_frag_list(vif, skb)) {
+                               if (net_ratelimit())
+                                       netdev_err(vif->dev,
+                                                  "Not enough memory to consolidate frag_list!\n");
+                               skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
+                               kfree_skb(skb);
+                               continue;
+                       }
+               }
+
                if (skb_is_nonlinear(skb) && skb_headlen(skb) < PKT_PROT_LEN) {
                        int target = min_t(int, skb->len, PKT_PROT_LEN);
                        __pskb_pull_tail(skb, target - skb_headlen(skb));