tunnels: Optimize tx path

We currently dirty a cache line to update tunnel device stats
(tx_packets/tx_bytes). We better use the txq->tx_bytes/tx_packets
counters that already are present in cpu cache, in the cache
line shared with txq->_xmit_lock

This patch extends IPTUNNEL_XMIT() macro to use txq pointer
provided by the caller.

Also &tunnel->dev->stats can be replaced by &dev->stats

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/ipip.h b/include/net/ipip.h
index 87acf8f3a15527cd0446e1dd6f75bdd60a636ba4..0159221a85093074f26fcbdbd487f0431504a28a 100644 (file)
--- a/include/net/ipip.h
+++ b/include/net/ipip.h
@@ -42,9 +42,9 @@ struct ip_tunnel_prl_entry
        ip_select_ident(iph, &rt->u.dst, NULL);                         \
                                                                        \
        err = ip_local_out(skb);                                        \
-       if (net_xmit_eval(err) == 0) {                                  \
-               stats->tx_bytes += pkt_len;                             \
-               stats->tx_packets++;                                    \
+       if (likely(net_xmit_eval(err) == 0)) {                          \
+               txq->tx_bytes += pkt_len;                               \
+               txq->tx_packets++;                                      \
        } else {                                                        \
                stats->tx_errors++;                                     \
                stats->tx_aborted_errors++;                             \

diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c
index 41ada9904d3125c32b063b3dad025cc4898c2c60..89ff9d5b1500cf90c7b8bfbe801edec8ac303549 100644 (file)
--- a/net/ipv4/ip_gre.c
+++ b/net/ipv4/ip_gre.c
@@ -662,7 +662,8 @@ drop_nolock:
 static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev)
 {
        struct ip_tunnel *tunnel = netdev_priv(dev);
-       struct net_device_stats *stats = &tunnel->dev->stats;
+       struct net_device_stats *stats = &dev->stats;
+       struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
        struct iphdr  *old_iph = ip_hdr(skb);
        struct iphdr  *tiph;
        u8     tos;
@@ -810,7 +811,7 @@ static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev
                struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom);
                if (!new_skb) {
                        ip_rt_put(rt);
-                       stats->tx_dropped++;
+                       txq->tx_dropped++;
                        dev_kfree_skb(skb);
                        return NETDEV_TX_OK;
                }

diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c
index 08ccd344de7a9a88d9f5236bb319bab86819e544..6a5539236ab37bac6938e9a9fa4ea85de15ce3ba 100644 (file)
--- a/net/ipv4/ipip.c
+++ b/net/ipv4/ipip.c
@@ -390,7 +390,8 @@ static int ipip_rcv(struct sk_buff *skb)
 static netdev_tx_t ipip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev)
 {
        struct ip_tunnel *tunnel = netdev_priv(dev);
-       struct net_device_stats *stats = &tunnel->dev->stats;
+       struct net_device_stats *stats = &dev->stats;
+       struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
        struct iphdr  *tiph = &tunnel->parms.iph;
        u8     tos = tunnel->parms.iph.tos;
        __be16 df = tiph->frag_off;
@@ -478,7 +479,7 @@ static netdev_tx_t ipip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev)
                struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom);
                if (!new_skb) {
                        ip_rt_put(rt);
-                       stats->tx_dropped++;
+                       txq->tx_dropped++;
                        dev_kfree_skb(skb);
                        return NETDEV_TX_OK;
                }

diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
index dbd19a78ca73ad32b048b8e9c8b89f3600195c90..99da272951dc23f19342e646ae6ee77fc813433a 100644 (file)
--- a/net/ipv6/sit.c
+++ b/net/ipv6/sit.c
@@ -555,7 +555,8 @@ static netdev_tx_t ipip6_tunnel_xmit(struct sk_buff *skb,
                                     struct net_device *dev)
 {
        struct ip_tunnel *tunnel = netdev_priv(dev);
-       struct net_device_stats *stats = &tunnel->dev->stats;
+       struct net_device_stats *stats = &dev->stats;
+       struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
        struct iphdr  *tiph = &tunnel->parms.iph;
        struct ipv6hdr *iph6 = ipv6_hdr(skb);
        u8     tos = tunnel->parms.iph.tos;
@@ -688,7 +689,7 @@ static netdev_tx_t ipip6_tunnel_xmit(struct sk_buff *skb,
                struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom);
                if (!new_skb) {
                        ip_rt_put(rt);
-                       stats->tx_dropped++;
+                       txq->tx_dropped++;
                        dev_kfree_skb(skb);
                        return NETDEV_TX_OK;
                }