iscsi-target: Prepare login code for multi-plexing support

This patch prepares the iscsi-target login code for multi-plexing
support.  This includes:

 - Adding iscsi_tpg_np->tpg_np_kref + iscsit_login_kref_put() for
   handling callback of iscsi_tpg_np->tpg_np_comp
 - Adding kref_put() in iscsit_deaccess_np()
 - Adding kref_put() and wait_for_completion() in
   iscsit_reset_np_thread()
 - Refactor login failure path release logic into
   iscsi_target_login_sess_out()
 - Update __iscsi_target_login_thread() to handle
   iscsi_post_login_handler() asynchronous completion
 - Add shutdown parameter for iscsit_clear_tpg_np_login_thread*()

v3 changes:
 - Convert iscsi_portal_group->np_login_lock to ->np_login_sem
 - Add LOGIN_FLAGS definitions

v2 changes:
 - Remove duplicate call to iscsi_post_login_handler() in
   __iscsi_target_login_thread()
 - Drop unused iscsi_np->np_login_tpg

Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>

diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index c4aeac314b2e894d390554a15ab8b3a280d2247f..1dff3e01f92c9becdb52b8295d28d21babb9c657 100644 (file)
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -220,11 +220,6 @@ int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
                spin_unlock_bh(&np->np_thread_lock);
                return -1;
        }
-       if (np->np_login_tpg) {
-               pr_err("np->np_login_tpg() is not NULL!\n");
-               spin_unlock_bh(&np->np_thread_lock);
-               return -1;
-       }
        spin_unlock_bh(&np->np_thread_lock);
        /*
         * Determine if the portal group is accepting storage traffic.
@@ -239,26 +234,38 @@ int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
        /*
         * Here we serialize access across the TIQN+TPG Tuple.
         */
-       ret = mutex_lock_interruptible(&tpg->np_login_lock);
+       ret = down_interruptible(&tpg->np_login_sem);
        if ((ret != 0) || signal_pending(current))
                return -1;
 
-       spin_lock_bh(&np->np_thread_lock);
-       np->np_login_tpg = tpg;
-       spin_unlock_bh(&np->np_thread_lock);
+       spin_lock_bh(&tpg->tpg_state_lock);
+       if (tpg->tpg_state != TPG_STATE_ACTIVE) {
+               spin_unlock_bh(&tpg->tpg_state_lock);
+               up(&tpg->np_login_sem);
+               return -1;
+       }
+       spin_unlock_bh(&tpg->tpg_state_lock);
 
        return 0;
 }
 
-int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
+void iscsit_login_kref_put(struct kref *kref)
+{
+       struct iscsi_tpg_np *tpg_np = container_of(kref,
+                               struct iscsi_tpg_np, tpg_np_kref);
+
+       complete(&tpg_np->tpg_np_comp);
+}
+
+int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg,
+                      struct iscsi_tpg_np *tpg_np)
 {
        struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 
-       spin_lock_bh(&np->np_thread_lock);
-       np->np_login_tpg = NULL;
-       spin_unlock_bh(&np->np_thread_lock);
+       up(&tpg->np_login_sem);
 
-       mutex_unlock(&tpg->np_login_lock);
+       if (tpg_np)
+               kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
 
        if (tiqn)
                iscsit_put_tiqn_for_login(tiqn);
@@ -410,20 +417,10 @@ struct iscsi_np *iscsit_add_np(
 int iscsit_reset_np_thread(
        struct iscsi_np *np,
        struct iscsi_tpg_np *tpg_np,
-       struct iscsi_portal_group *tpg)
+       struct iscsi_portal_group *tpg,
+       bool shutdown)
 {
        spin_lock_bh(&np->np_thread_lock);
-       if (tpg && tpg_np) {
-               /*
-                * The reset operation need only be performed when the
-                * passed struct iscsi_portal_group has a login in progress
-                * to one of the network portals.
-                */
-               if (tpg_np->tpg_np->np_login_tpg != tpg) {
-                       spin_unlock_bh(&np->np_thread_lock);
-                       return 0;
-               }
-       }
        if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
                spin_unlock_bh(&np->np_thread_lock);
                return 0;
@@ -438,6 +435,12 @@ int iscsit_reset_np_thread(
        }
        spin_unlock_bh(&np->np_thread_lock);
 
+       if (tpg_np && shutdown) {
+               kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
+
+               wait_for_completion(&tpg_np->tpg_np_comp);
+       }
+
        return 0;
 }
 

diff --git a/drivers/target/iscsi/iscsi_target.h b/drivers/target/iscsi/iscsi_target.h
index 2c437cb8ca00eedfd05f6ff8747f42acc5195375..f82f6273526e77fa845eeeb02e989bc82bb6d11c 100644 (file)
--- a/drivers/target/iscsi/iscsi_target.h
+++ b/drivers/target/iscsi/iscsi_target.h
@@ -7,13 +7,15 @@ extern void iscsit_put_tiqn_for_login(struct iscsi_tiqn *);
 extern struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *);
 extern void iscsit_del_tiqn(struct iscsi_tiqn *);
 extern int iscsit_access_np(struct iscsi_np *, struct iscsi_portal_group *);
-extern int iscsit_deaccess_np(struct iscsi_np *, struct iscsi_portal_group *);
+extern void iscsit_login_kref_put(struct kref *);
+extern int iscsit_deaccess_np(struct iscsi_np *, struct iscsi_portal_group *,
+                               struct iscsi_tpg_np *);
 extern bool iscsit_check_np_match(struct __kernel_sockaddr_storage *,
                                struct iscsi_np *, int);
 extern struct iscsi_np *iscsit_add_np(struct __kernel_sockaddr_storage *,
                                char *, int);
 extern int iscsit_reset_np_thread(struct iscsi_np *, struct iscsi_tpg_np *,
-                               struct iscsi_portal_group *);
+                               struct iscsi_portal_group *, bool);
 extern int iscsit_del_np(struct iscsi_np *);
 extern int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8, unsigned char *);
 extern void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *);

diff --git a/drivers/target/iscsi/iscsi_target_core.h b/drivers/target/iscsi/iscsi_target_core.h
index 4f77a78edef9dbfc26f218ca3e6147e986a5ebc8..46228465d1903b7f79e648460cf7e46ad177771e 100644 (file)
--- a/drivers/target/iscsi/iscsi_target_core.h
+++ b/drivers/target/iscsi/iscsi_target_core.h
@@ -554,6 +554,13 @@ struct iscsi_conn {
        struct completion       rx_half_close_comp;
        /* socket used by this connection */
        struct socket           *sock;
+       void                    (*orig_data_ready)(struct sock *, int);
+#define LOGIN_FLAGS_READ_ACTIVE                1
+#define LOGIN_FLAGS_CLOSED             2
+#define LOGIN_FLAGS_READY              4
+       unsigned long           login_flags;
+       struct delayed_work     login_work;
+       struct iscsi_login      *login;
        struct timer_list       nopin_timer;
        struct timer_list       nopin_response_timer;
        struct timer_list       transport_timer;
@@ -584,6 +591,7 @@ struct iscsi_conn {
        void                    *context;
        struct iscsi_login_thread_s *login_thread;
        struct iscsi_portal_group *tpg;
+       struct iscsi_tpg_np     *tpg_np;
        /* Pointer to parent session */
        struct iscsi_session    *sess;
        /* Pointer to thread_set in use for this conn's threads */
@@ -682,6 +690,7 @@ struct iscsi_login {
        u8 version_max;
        u8 login_complete;
        u8 login_failed;
+       bool zero_tsih;
        char isid[6];
        u32 cmd_sn;
        itt_t init_task_tag;
@@ -694,6 +703,7 @@ struct iscsi_login {
        char *req_buf;
        char *rsp_buf;
        struct iscsi_conn *conn;
+       struct iscsi_np *np;
 } ____cacheline_aligned;
 
 struct iscsi_node_attrib {
@@ -773,7 +783,6 @@ struct iscsi_np {
        struct __kernel_sockaddr_storage np_sockaddr;
        struct task_struct      *np_thread;
        struct timer_list       np_login_timer;
-       struct iscsi_portal_group *np_login_tpg;
        void                    *np_context;
        struct iscsit_transport *np_transport;
        struct list_head        np_list;
@@ -788,6 +797,8 @@ struct iscsi_tpg_np {
        struct list_head        tpg_np_parent_list;
        struct se_tpg_np        se_tpg_np;
        spinlock_t              tpg_np_parent_lock;
+       struct completion       tpg_np_comp;
+       struct kref             tpg_np_kref;
 };
 
 struct iscsi_portal_group {
@@ -809,7 +820,7 @@ struct iscsi_portal_group {
        spinlock_t              tpg_state_lock;
        struct se_portal_group tpg_se_tpg;
        struct mutex            tpg_access_lock;
-       struct mutex            np_login_lock;
+       struct semaphore        np_login_sem;
        struct iscsi_tpg_attrib tpg_attrib;
        struct iscsi_node_auth  tpg_demo_auth;
        /* Pointer to default list of iSCSI parameters for TPG */

diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
index 0e85238bdf4829eb51d83375cbbfd4dcbcad882d..d903c0c702d6c2e44ffa29efb1978196df9e2f1a 100644 (file)
--- a/drivers/target/iscsi/iscsi_target_login.c
+++ b/drivers/target/iscsi/iscsi_target_login.c
@@ -50,6 +50,7 @@ static struct iscsi_login *iscsi_login_init_conn(struct iscsi_conn *conn)
                pr_err("Unable to allocate memory for struct iscsi_login.\n");
                return NULL;
        }
+       conn->login = login;
        login->conn = conn;
        login->first_request = 1;
 
@@ -684,7 +685,7 @@ static void iscsi_post_login_start_timers(struct iscsi_conn *conn)
                iscsit_start_nopin_timer(conn);
 }
 
-static int iscsi_post_login_handler(
+int iscsi_post_login_handler(
        struct iscsi_np *np,
        struct iscsi_conn *conn,
        u8 zero_tsih)
@@ -1124,6 +1125,77 @@ iscsit_conn_set_transport(struct iscsi_conn *conn, struct iscsit_transport *t)
        return 0;
 }
 
+void iscsi_target_login_sess_out(struct iscsi_conn *conn,
+               struct iscsi_np *np, bool zero_tsih, bool new_sess)
+{
+       if (new_sess == false)
+               goto old_sess_out;
+
+       pr_err("iSCSI Login negotiation failed.\n");
+       iscsit_collect_login_stats(conn, ISCSI_STATUS_CLS_INITIATOR_ERR,
+                                  ISCSI_LOGIN_STATUS_INIT_ERR);
+       if (!zero_tsih || !conn->sess)
+               goto old_sess_out;
+       if (conn->sess->se_sess)
+               transport_free_session(conn->sess->se_sess);
+       if (conn->sess->session_index != 0) {
+               spin_lock_bh(&sess_idr_lock);
+               idr_remove(&sess_idr, conn->sess->session_index);
+               spin_unlock_bh(&sess_idr_lock);
+       }
+       kfree(conn->sess->sess_ops);
+       kfree(conn->sess);
+
+old_sess_out:
+       iscsi_stop_login_thread_timer(np);
+       /*
+        * If login negotiation fails check if the Time2Retain timer
+        * needs to be restarted.
+        */
+       if (!zero_tsih && conn->sess) {
+               spin_lock_bh(&conn->sess->conn_lock);
+               if (conn->sess->session_state == TARG_SESS_STATE_FAILED) {
+                       struct se_portal_group *se_tpg =
+                                       &ISCSI_TPG_C(conn)->tpg_se_tpg;
+
+                       atomic_set(&conn->sess->session_continuation, 0);
+                       spin_unlock_bh(&conn->sess->conn_lock);
+                       spin_lock_bh(&se_tpg->session_lock);
+                       iscsit_start_time2retain_handler(conn->sess);
+                       spin_unlock_bh(&se_tpg->session_lock);
+               } else
+                       spin_unlock_bh(&conn->sess->conn_lock);
+               iscsit_dec_session_usage_count(conn->sess);
+       }
+
+       if (!IS_ERR(conn->conn_rx_hash.tfm))
+               crypto_free_hash(conn->conn_rx_hash.tfm);
+       if (!IS_ERR(conn->conn_tx_hash.tfm))
+               crypto_free_hash(conn->conn_tx_hash.tfm);
+
+       if (conn->conn_cpumask)
+               free_cpumask_var(conn->conn_cpumask);
+
+       kfree(conn->conn_ops);
+
+       if (conn->param_list) {
+               iscsi_release_param_list(conn->param_list);
+               conn->param_list = NULL;
+       }
+       iscsi_target_nego_release(conn);
+
+       if (conn->sock) {
+               sock_release(conn->sock);
+               conn->sock = NULL;
+       }
+
+       if (conn->conn_transport->iscsit_free_conn)
+               conn->conn_transport->iscsit_free_conn(conn);
+
+       iscsit_put_transport(conn->conn_transport);
+       kfree(conn);
+}
+
 static int __iscsi_target_login_thread(struct iscsi_np *np)
 {
        u8 *buffer, zero_tsih = 0;
@@ -1132,6 +1204,8 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
        struct iscsi_login *login;
        struct iscsi_portal_group *tpg = NULL;
        struct iscsi_login_req *pdu;
+       struct iscsi_tpg_np *tpg_np;
+       bool new_sess = false;
 
        flush_signals(current);
 
@@ -1273,6 +1347,7 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
                tpg = conn->tpg;
                goto new_sess_out;
        }
+       login->zero_tsih = zero_tsih;
 
        tpg = conn->tpg;
        if (!tpg) {
@@ -1288,7 +1363,8 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
                        goto old_sess_out;
        }
 
-       if (iscsi_target_start_negotiation(login, conn) < 0)
+       ret = iscsi_target_start_negotiation(login, conn);
+       if (ret < 0)
                goto new_sess_out;
 
        if (!conn->sess) {
@@ -1301,84 +1377,32 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
        if (signal_pending(current))
                goto new_sess_out;
 
-       ret = iscsi_post_login_handler(np, conn, zero_tsih);
+       if (ret == 1) {
+               tpg_np = conn->tpg_np;
 
-       if (ret < 0)
-               goto new_sess_out;
+               ret = iscsi_post_login_handler(np, conn, zero_tsih);
+               if (ret < 0)
+                       goto new_sess_out;
+
+               iscsit_deaccess_np(np, tpg, tpg_np);
+       }
 
-       iscsit_deaccess_np(np, tpg);
        tpg = NULL;
+       tpg_np = NULL;
        /* Get another socket */
        return 1;
 
 new_sess_out:
-       pr_err("iSCSI Login negotiation failed.\n");
-       iscsit_collect_login_stats(conn, ISCSI_STATUS_CLS_INITIATOR_ERR,
-                                 ISCSI_LOGIN_STATUS_INIT_ERR);
-       if (!zero_tsih || !conn->sess)
-               goto old_sess_out;
-       if (conn->sess->se_sess)
-               transport_free_session(conn->sess->se_sess);
-       if (conn->sess->session_index != 0) {
-               spin_lock_bh(&sess_idr_lock);
-               idr_remove(&sess_idr, conn->sess->session_index);
-               spin_unlock_bh(&sess_idr_lock);
-       }
-       kfree(conn->sess->sess_ops);
-       kfree(conn->sess);
+       new_sess = true;
 old_sess_out:
-       iscsi_stop_login_thread_timer(np);
-       /*
-        * If login negotiation fails check if the Time2Retain timer
-        * needs to be restarted.
-        */
-       if (!zero_tsih && conn->sess) {
-               spin_lock_bh(&conn->sess->conn_lock);
-               if (conn->sess->session_state == TARG_SESS_STATE_FAILED) {
-                       struct se_portal_group *se_tpg =
-                                       &ISCSI_TPG_C(conn)->tpg_se_tpg;
-
-                       atomic_set(&conn->sess->session_continuation, 0);
-                       spin_unlock_bh(&conn->sess->conn_lock);
-                       spin_lock_bh(&se_tpg->session_lock);
-                       iscsit_start_time2retain_handler(conn->sess);
-                       spin_unlock_bh(&se_tpg->session_lock);
-               } else
-                       spin_unlock_bh(&conn->sess->conn_lock);
-               iscsit_dec_session_usage_count(conn->sess);
-       }
-
-       if (!IS_ERR(conn->conn_rx_hash.tfm))
-               crypto_free_hash(conn->conn_rx_hash.tfm);
-       if (!IS_ERR(conn->conn_tx_hash.tfm))
-               crypto_free_hash(conn->conn_tx_hash.tfm);
-
-       if (conn->conn_cpumask)
-               free_cpumask_var(conn->conn_cpumask);
-
-       kfree(conn->conn_ops);
-
-       if (conn->param_list) {
-               iscsi_release_param_list(conn->param_list);
-               conn->param_list = NULL;
-       }
-       iscsi_target_nego_release(conn);
-
-       if (conn->sock) {
-               sock_release(conn->sock);
-               conn->sock = NULL;
-       }
-
-       if (conn->conn_transport->iscsit_free_conn)
-               conn->conn_transport->iscsit_free_conn(conn);
-
-       iscsit_put_transport(conn->conn_transport);
-
-       kfree(conn);
+       tpg_np = conn->tpg_np;
+       iscsi_target_login_sess_out(conn, np, zero_tsih, new_sess);
+       new_sess = false;
 
        if (tpg) {
-               iscsit_deaccess_np(np, tpg);
+               iscsit_deaccess_np(np, tpg, tpg_np);
                tpg = NULL;
+               tpg_np = NULL;
        }
 
 out:

diff --git a/drivers/target/iscsi/iscsi_target_login.h b/drivers/target/iscsi/iscsi_target_login.h
index 63efd2878451717823a5835f213e524c3b4e6460..29d098324b7f947e553eade28ad2ccbecc8b887b 100644 (file)
--- a/drivers/target/iscsi/iscsi_target_login.h
+++ b/drivers/target/iscsi/iscsi_target_login.h
@@ -12,6 +12,9 @@ extern int iscsit_accept_np(struct iscsi_np *, struct iscsi_conn *);
 extern int iscsit_get_login_rx(struct iscsi_conn *, struct iscsi_login *);
 extern int iscsit_put_login_tx(struct iscsi_conn *, struct iscsi_login *, u32);
 extern void iscsit_free_conn(struct iscsi_np *, struct iscsi_conn *);
+extern int iscsi_post_login_handler(struct iscsi_np *, struct iscsi_conn *, u8);
+extern void iscsi_target_login_sess_out(struct iscsi_conn *, struct iscsi_np *,
+                               bool, bool);
 extern int iscsi_target_login_thread(void *);
 extern int iscsi_login_disable_FIM_keys(struct iscsi_param_list *, struct iscsi_conn *);
 

diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
index 439260b7d87fa69a7e379c66e4474c416e35a21d..4bf8913992a50c26e93c7ee00e6384a9ecb6a112 100644 (file)
--- a/drivers/target/iscsi/iscsi_target_tpg.c
+++ b/drivers/target/iscsi/iscsi_target_tpg.c
@@ -49,7 +49,7 @@ struct iscsi_portal_group *iscsit_alloc_portal_group(struct iscsi_tiqn *tiqn, u1
        INIT_LIST_HEAD(&tpg->tpg_gnp_list);
        INIT_LIST_HEAD(&tpg->tpg_list);
        mutex_init(&tpg->tpg_access_lock);
-       mutex_init(&tpg->np_login_lock);
+       sema_init(&tpg->np_login_sem, 1);
        spin_lock_init(&tpg->tpg_state_lock);
        spin_lock_init(&tpg->tpg_np_lock);
 
@@ -175,18 +175,20 @@ void iscsit_put_tpg(struct iscsi_portal_group *tpg)
 
 static void iscsit_clear_tpg_np_login_thread(
        struct iscsi_tpg_np *tpg_np,
-       struct iscsi_portal_group *tpg)
+       struct iscsi_portal_group *tpg,
+       bool shutdown)
 {
        if (!tpg_np->tpg_np) {
                pr_err("struct iscsi_tpg_np->tpg_np is NULL!\n");
                return;
        }
 
-       iscsit_reset_np_thread(tpg_np->tpg_np, tpg_np, tpg);
+       iscsit_reset_np_thread(tpg_np->tpg_np, tpg_np, tpg, shutdown);
 }
 
 void iscsit_clear_tpg_np_login_threads(
-       struct iscsi_portal_group *tpg)
+       struct iscsi_portal_group *tpg,
+       bool shutdown)
 {
        struct iscsi_tpg_np *tpg_np;
 
@@ -197,7 +199,7 @@ void iscsit_clear_tpg_np_login_threads(
                        continue;
                }
                spin_unlock(&tpg->tpg_np_lock);
-               iscsit_clear_tpg_np_login_thread(tpg_np, tpg);
+               iscsit_clear_tpg_np_login_thread(tpg_np, tpg, shutdown);
                spin_lock(&tpg->tpg_np_lock);
        }
        spin_unlock(&tpg->tpg_np_lock);
@@ -268,6 +270,8 @@ int iscsit_tpg_del_portal_group(
        tpg->tpg_state = TPG_STATE_INACTIVE;
        spin_unlock(&tpg->tpg_state_lock);
 
+       iscsit_clear_tpg_np_login_threads(tpg, true);
+
        if (iscsit_release_sessions_for_tpg(tpg, force) < 0) {
                pr_err("Unable to delete iSCSI Target Portal Group:"
                        " %hu while active sessions exist, and force=0\n",
@@ -368,7 +372,7 @@ int iscsit_tpg_disable_portal_group(struct iscsi_portal_group *tpg, int force)
        tpg->tpg_state = TPG_STATE_INACTIVE;
        spin_unlock(&tpg->tpg_state_lock);
 
-       iscsit_clear_tpg_np_login_threads(tpg);
+       iscsit_clear_tpg_np_login_threads(tpg, false);
 
        if (iscsit_release_sessions_for_tpg(tpg, force) < 0) {
                spin_lock(&tpg->tpg_state_lock);
@@ -520,7 +524,7 @@ static int iscsit_tpg_release_np(
        struct iscsi_portal_group *tpg,
        struct iscsi_np *np)
 {
-       iscsit_clear_tpg_np_login_thread(tpg_np, tpg);
+       iscsit_clear_tpg_np_login_thread(tpg_np, tpg, true);
 
        pr_debug("CORE[%s] - Removed Network Portal: %s:%hu,%hu on %s\n",
                tpg->tpg_tiqn->tiqn, np->np_ip, np->np_port, tpg->tpgt,

diff --git a/drivers/target/iscsi/iscsi_target_tpg.h b/drivers/target/iscsi/iscsi_target_tpg.h
index dda48c141a8c8613a6c2172a9f958bd3332372af..60ef4a9ad91b045c587fa38adc6db8dc9820e0c1 100644 (file)
--- a/drivers/target/iscsi/iscsi_target_tpg.h
+++ b/drivers/target/iscsi/iscsi_target_tpg.h
@@ -8,7 +8,7 @@ extern struct iscsi_portal_group *iscsit_get_tpg_from_np(struct iscsi_tiqn *,
                        struct iscsi_np *);
 extern int iscsit_get_tpg(struct iscsi_portal_group *);
 extern void iscsit_put_tpg(struct iscsi_portal_group *);
-extern void iscsit_clear_tpg_np_login_threads(struct iscsi_portal_group *);
+extern void iscsit_clear_tpg_np_login_threads(struct iscsi_portal_group *, bool);
 extern void iscsit_tpg_dump_params(struct iscsi_portal_group *);
 extern int iscsit_tpg_add_portal_group(struct iscsi_tiqn *, struct iscsi_portal_group *);
 extern int iscsit_tpg_del_portal_group(struct iscsi_tiqn *, struct iscsi_portal_group *,