net/smc: unregister rkeys of unused buffer

When an rmb is no longer in use by a connection, unregister its rkey at
the remote peer with an LLC DELETE RKEY message. With this change,
unused buffers held in the buffer pool are no longer registered at the
remote peer. They are registered before the buffer is actually used and
unregistered when they are no longer used by a connection.

Signed-off-by: Karsten Graul <kgraul@linux.ibm.com>
Signed-off-by: Ursula Braun <ubraun@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 7657e249f526c65e6841d4d8c023fb47a5fb5f2c..4b865250e238e04bde688259c3559e04330feca0 100644 (file)
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -299,14 +299,17 @@ static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
        smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 }
 
-/* register a new rmb, optionally send confirm_rkey msg to register with peer */
+/* register a new rmb, send confirm_rkey msg to register with peer */
 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
                       bool conf_rkey)
 {
-       /* register memory region for new rmb */
-       if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
-               rmb_desc->regerr = 1;
-               return -EFAULT;
+       if (!rmb_desc->wr_reg) {
+               /* register memory region for new rmb */
+               if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
+                       rmb_desc->regerr = 1;
+                       return -EFAULT;
+               }
+               rmb_desc->wr_reg = 1;
        }
        if (!conf_rkey)
                return 0;
@@ -581,8 +584,7 @@ static int smc_connect_rdma(struct smc_sock *smc,
                        return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
                                                 local_contact);
        } else {
-               if (!smc->conn.rmb_desc->reused &&
-                   smc_reg_rmb(link, smc->conn.rmb_desc, true))
+               if (smc_reg_rmb(link, smc->conn.rmb_desc, true))
                        return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
                                                 local_contact);
        }
@@ -1143,10 +1145,8 @@ static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
        struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
 
        if (local_contact != SMC_FIRST_CONTACT) {
-               if (!new_smc->conn.rmb_desc->reused) {
-                       if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
-                               return SMC_CLC_DECL_ERR_REGRMB;
-               }
+               if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
+                       return SMC_CLC_DECL_ERR_REGRMB;
        }
        smc_rmb_sync_sg_for_device(&new_smc->conn);
 

diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index ec7a7ed3b968fb0cc8e9552f7efbc833c38735e5..1382ddae591e9ccac9bb48087ee6f4c80d9de1bb 100644 (file)
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -298,8 +298,13 @@ static void smc_buf_unuse(struct smc_connection *conn,
                conn->sndbuf_desc->used = 0;
        if (conn->rmb_desc) {
                if (!conn->rmb_desc->regerr) {
-                       conn->rmb_desc->reused = 1;
                        conn->rmb_desc->used = 0;
+                       if (!lgr->is_smcd) {
+                               /* unregister rmb with peer */
+                               smc_llc_do_delete_rkey(
+                                               &lgr->lnk[SMC_SINGLE_LINK],
+                                               conn->rmb_desc);
+                       }
                } else {
                        /* buf registration failed, reuse not possible */
                        write_lock_bh(&lgr->rmbs_lock);

diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index bce39d6df45a4fb2f9f3a47ba06cc8dc351e2a66..e177c6675038c99a9eb1a193630bf74dd87cfd72 100644 (file)
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -130,7 +130,7 @@ struct smc_buf_desc {
        struct page             *pages;
        int                     len;            /* length of buffer */
        u32                     used;           /* currently used / unused */
-       u8                      reused  : 1;    /* new created / reused */
+       u8                      wr_reg  : 1;    /* mem region registered */
        u8                      regerr  : 1;    /* err during registration */
        union {
                struct { /* SMC-R */