net/smc: add smcr_port_err() and smcr_link_down() processing

Call smcr_port_err() when an IB event reports an inactive IB device.
smcr_port_err() calls smcr_link_down() for all affected links.
smcr_link_down() either triggers the local DELETE_LINK processing, or
sends an DELETE_LINK LLC message to the SMC server to initiate the
processing.
The old handler function smc_port_terminate() is removed.
Add helper smcr_link_down_cond() to take a link down conditionally, and
smcr_link_down_cond_sched() to schedule the link_down processing to a
work.

Signed-off-by: Karsten Graul <kgraul@linux.ibm.com>
Reviewed-by: Ursula Braun <ubraun@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index 20bc9e46bf520e08a77f163996a2320489c70a28..62108e0cd529fdfcf894045f5b667017098864c8 100644 (file)
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -56,6 +56,7 @@ static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
 static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft);
 
 static void smc_link_up_work(struct work_struct *work);
+static void smc_link_down_work(struct work_struct *work);
 
 /* return head of link group list and its lock for a given link group */
 static inline struct list_head *smc_lgr_list_head(struct smc_link_group *lgr,
@@ -320,6 +321,7 @@ static int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk,
        lnk->smcibdev = ini->ib_dev;
        lnk->ibport = ini->ib_port;
        lnk->path_mtu = ini->ib_dev->pattr[ini->ib_port - 1].active_mtu;
+       INIT_WORK(&lnk->link_down_wrk, smc_link_down_work);
        if (!ini->ib_dev->initialized) {
                rc = (int)smc_ib_setup_per_ibdev(ini->ib_dev);
                if (rc)
@@ -818,36 +820,6 @@ void smc_lgr_terminate_sched(struct smc_link_group *lgr)
        schedule_work(&lgr->terminate_work);
 }
 
-/* Called when IB port is terminated */
-void smc_port_terminate(struct smc_ib_device *smcibdev, u8 ibport)
-{
-       struct smc_link_group *lgr, *l;
-       LIST_HEAD(lgr_free_list);
-       int i;
-
-       spin_lock_bh(&smc_lgr_list.lock);
-       list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {
-               if (lgr->is_smcd)
-                       continue;
-               /* tbd - terminate only when no more links are active */
-               for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
-                       if (!smc_link_usable(&lgr->lnk[i]))
-                               continue;
-                       if (lgr->lnk[i].smcibdev == smcibdev &&
-                           lgr->lnk[i].ibport == ibport) {
-                               list_move(&lgr->list, &lgr_free_list);
-                               lgr->freeing = 1;
-                       }
-               }
-       }
-       spin_unlock_bh(&smc_lgr_list.lock);
-
-       list_for_each_entry_safe(lgr, l, &lgr_free_list, list) {
-               list_del_init(&lgr->list);
-               __smc_lgr_terminate(lgr, false);
-       }
-}
-
 /* Called when peer lgr shutdown (regularly or abnormally) is received */
 void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid, unsigned short vlan)
 {
@@ -1000,6 +972,79 @@ void smcr_port_add(struct smc_ib_device *smcibdev, u8 ibport)
        }
 }
 
+/* link is down - switch connections to alternate link,
+ * must be called under lgr->llc_conf_mutex lock
+ */
+static void smcr_link_down(struct smc_link *lnk)
+{
+       struct smc_link_group *lgr = lnk->lgr;
+       struct smc_link *to_lnk;
+       int del_link_id;
+
+       if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list))
+               return;
+
+       smc_ib_modify_qp_reset(lnk);
+       to_lnk = NULL;
+       /* tbd: call to_lnk = smc_switch_conns(lgr, lnk, true); */
+       if (!to_lnk) { /* no backup link available */
+               smcr_link_clear(lnk);
+               return;
+       }
+       lgr->type = SMC_LGR_SINGLE;
+       del_link_id = lnk->link_id;
+
+       if (lgr->role == SMC_SERV) {
+               /* trigger local delete link processing */
+       } else {
+               if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
+                       /* another llc task is ongoing */
+                       mutex_unlock(&lgr->llc_conf_mutex);
+                       wait_event_interruptible_timeout(lgr->llc_waiter,
+                               (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE),
+                               SMC_LLC_WAIT_TIME);
+                       mutex_lock(&lgr->llc_conf_mutex);
+               }
+               smc_llc_send_delete_link(to_lnk, del_link_id, SMC_LLC_REQ, true,
+                                        SMC_LLC_DEL_LOST_PATH);
+       }
+}
+
+/* must be called under lgr->llc_conf_mutex lock */
+void smcr_link_down_cond(struct smc_link *lnk)
+{
+       if (smc_link_downing(&lnk->state))
+               smcr_link_down(lnk);
+}
+
+/* will get the lgr->llc_conf_mutex lock */
+void smcr_link_down_cond_sched(struct smc_link *lnk)
+{
+       if (smc_link_downing(&lnk->state))
+               schedule_work(&lnk->link_down_wrk);
+}
+
+void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport)
+{
+       struct smc_link_group *lgr, *n;
+       int i;
+
+       list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) {
+               if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id,
+                           SMC_MAX_PNETID_LEN))
+                       continue; /* lgr is not affected */
+               if (list_empty(&lgr->list))
+                       continue;
+               for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
+                       struct smc_link *lnk = &lgr->lnk[i];
+
+                       if (smc_link_usable(lnk) &&
+                           lnk->smcibdev == smcibdev && lnk->ibport == ibport)
+                               smcr_link_down_cond_sched(lnk);
+               }
+       }
+}
+
 static void smc_link_up_work(struct work_struct *work)
 {
        struct smc_ib_up_work *ib_work = container_of(work,
@@ -1014,6 +1059,20 @@ out:
        kfree(ib_work);
 }
 
+static void smc_link_down_work(struct work_struct *work)
+{
+       struct smc_link *link = container_of(work, struct smc_link,
+                                            link_down_wrk);
+       struct smc_link_group *lgr = link->lgr;
+
+       if (list_empty(&lgr->list))
+               return;
+       wake_up_interruptible_all(&lgr->llc_waiter);
+       mutex_lock(&lgr->llc_conf_mutex);
+       smcr_link_down(link);
+       mutex_unlock(&lgr->llc_conf_mutex);
+}
+
 /* Determine vlan of internal TCP socket.
  * @vlan_id: address to store the determined vlan id into
  */

diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index 86453ad834914c224f64a2e982327fee366b6aa5..da3cddbd16511079bab0a57e5843378462598a06 100644 (file)
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -117,6 +117,7 @@ struct smc_link {
        u8                      link_id;        /* unique # within link group */
        u8                      link_idx;       /* index in lgr link array */
        struct smc_link_group   *lgr;           /* parent link group */
+       struct work_struct      link_down_wrk;  /* wrk to bring link down */
 
        enum smc_link_state     state;          /* state of link */
        struct delayed_work     llc_testlink_wrk; /* testlink worker */
@@ -344,8 +345,8 @@ struct smc_clc_msg_local;
 void smc_lgr_forget(struct smc_link_group *lgr);
 void smc_lgr_cleanup_early(struct smc_connection *conn);
 void smc_lgr_terminate_sched(struct smc_link_group *lgr);
-void smc_port_terminate(struct smc_ib_device *smcibdev, u8 ibport);
 void smcr_port_add(struct smc_ib_device *smcibdev, u8 ibport);
+void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport);
 void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid,
                        unsigned short vlan);
 void smc_smcd_terminate_all(struct smcd_dev *dev);
@@ -376,6 +377,9 @@ void smcr_link_clear(struct smc_link *lnk);
 int smcr_buf_map_lgr(struct smc_link *lnk);
 int smcr_buf_reg_lgr(struct smc_link *lnk);
 int smcr_link_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc);
+void smcr_link_down_cond(struct smc_link *lnk);
+void smcr_link_down_cond_sched(struct smc_link *lnk);
+
 static inline struct smc_link_group *smc_get_lgr(struct smc_link *link)
 {
        return link->lgr;

diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c
index 545fb0bc3714c95f4d4ad55230002805c7c315b3..2c743caad69a12a590be327ebc846a5cf557a5db 100644 (file)
--- a/net/smc/smc_ib.c
+++ b/net/smc/smc_ib.c
@@ -249,7 +249,7 @@ static void smc_ib_port_event_work(struct work_struct *work)
                clear_bit(port_idx, &smcibdev->port_event_mask);
                if (!smc_ib_port_active(smcibdev, port_idx + 1)) {
                        set_bit(port_idx, smcibdev->ports_going_away);
-                       smc_port_terminate(smcibdev, port_idx + 1);
+                       smcr_port_err(smcibdev, port_idx + 1);
                } else {
                        clear_bit(port_idx, smcibdev->ports_going_away);
                        smcr_port_add(smcibdev, port_idx + 1);

diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h
index d2c50d3e43a6b26c5cfc9d175bd3d76d154232d6..4ed4486e50823dd0bd37f132497c8680de16c783 100644 (file)
--- a/net/smc/smc_llc.h
+++ b/net/smc/smc_llc.h
@@ -35,6 +35,9 @@ enum smc_llc_msg_type {
        SMC_LLC_DELETE_RKEY             = 0x09,
 };
 
+#define smc_link_downing(state) \
+       (cmpxchg(state, SMC_LNK_ACTIVE, SMC_LNK_INACTIVE) == SMC_LNK_ACTIVE)
+
 /* LLC DELETE LINK Request Reason Codes */
 #define SMC_LLC_DEL_LOST_PATH          0x00010000
 #define SMC_LLC_DEL_OP_INIT_TERM       0x00020000