net: manage napi add/del idempotence explicitly

To RCUify napi->dev_list we need to replace list_del_init()
with list_del_rcu(). There is no _init() version for RCU for
obvious reasons. Up until now netif_napi_del() was idempotent
so to make sure it remains such add a bit which is set when
NAPI is listed, and cleared when it removed. Since we don't
expect multiple calls to netif_napi_add() to be correct,
add a warning on that side.

Now that napi_hash_add / napi_hash_del are only called by
napi_add / del we can actually steal its bit. We just need
to make sure hash node is initialized correctly.

Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 74ed9521509101dbeda99ac5430bb5d410d9f854..157e0242e9ee564d61af6d310039958f81a56bca 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -355,7 +355,7 @@ enum {
        NAPI_STATE_MISSED,      /* reschedule a napi */
        NAPI_STATE_DISABLE,     /* Disable pending */
        NAPI_STATE_NPSVC,       /* Netpoll - don't dequeue from poll_list */
-       NAPI_STATE_HASHED,      /* In NAPI hash (busy polling possible) */
+       NAPI_STATE_LISTED,      /* NAPI added to system lists */
        NAPI_STATE_NO_BUSY_POLL,/* Do not add in napi_hash, no busy polling */
        NAPI_STATE_IN_BUSY_POLL,/* sk_busy_loop() owns this NAPI */
 };
@@ -365,7 +365,7 @@ enum {
        NAPIF_STATE_MISSED       = BIT(NAPI_STATE_MISSED),
        NAPIF_STATE_DISABLE      = BIT(NAPI_STATE_DISABLE),
        NAPIF_STATE_NPSVC        = BIT(NAPI_STATE_NPSVC),
-       NAPIF_STATE_HASHED       = BIT(NAPI_STATE_HASHED),
+       NAPIF_STATE_LISTED       = BIT(NAPI_STATE_LISTED),
        NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
        NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
 };

diff --git a/net/core/dev.c b/net/core/dev.c
index 205b7339d4ae7aca531449e941a319b78298fddc..e0a1be9868242a0ef5add51e30c98cdf3ae048e4 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6533,8 +6533,7 @@ EXPORT_SYMBOL(napi_busy_loop);
 
 static void napi_hash_add(struct napi_struct *napi)
 {
-       if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
-           test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
+       if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
                return;
 
        spin_lock(&napi_hash_lock);
@@ -6559,8 +6558,7 @@ static void napi_hash_del(struct napi_struct *napi)
 {
        spin_lock(&napi_hash_lock);
 
-       if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
-               hlist_del_rcu(&napi->napi_hash_node);
+       hlist_del_init_rcu(&napi->napi_hash_node);
 
        spin_unlock(&napi_hash_lock);
 }
@@ -6595,7 +6593,11 @@ static void init_gro_hash(struct napi_struct *napi)
 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
                    int (*poll)(struct napi_struct *, int), int weight)
 {
+       if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
+               return;
+
        INIT_LIST_HEAD(&napi->poll_list);
+       INIT_HLIST_NODE(&napi->napi_hash_node);
        hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
        napi->timer.function = napi_watchdog;
        init_gro_hash(napi);
@@ -6650,6 +6652,9 @@ static void flush_gro_hash(struct napi_struct *napi)
 /* Must be called in process context */
 void __netif_napi_del(struct napi_struct *napi)
 {
+       if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
+               return;
+
        napi_hash_del(napi);
        list_del_init(&napi->dev_list);
        napi_free_frags(napi);