iscsi: Support hot removal of LUN based on LUN open/close

When hot removal of a LUN is started, callback is called for each
iSCSI connection which accesses the LUN. Callback checks all transfer
tasks complete and then close the LUN.

If the connection clears all transfer tasks before getting all responses
to them from the initiator, the initiator continues to retry data write.
Hence the connection have to wait until all transfer tasks complete.

Change-Id: Iad9063673cfedbd78758890d55a4254512e4fca4
Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/417199
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

include/spdk/scsi.h

@@ -190,6 +190,15 @@ const char *spdk_scsi_lun_get_bdev_name(const struct spdk_scsi_lun *lun);
  */
 const struct spdk_scsi_dev *spdk_scsi_lun_get_dev(const struct spdk_scsi_lun *lun);
 
+/**
+ * Check if the logical unit is hot removing.
+ *
+ * \param lun Logical unit
+ *
+ * \return true if removing, false otherwise.
+ */
+bool spdk_scsi_lun_is_removing(const struct spdk_scsi_lun *lun);
+
 /**
  * Get the name of the given SCSI device.
  *

lib/iscsi/conn.c

@@ -50,7 +50,6 @@
 #include "iscsi/conn.h"
 #include "iscsi/tgt_node.h"
 #include "iscsi/portal_grp.h"
-#include "spdk/scsi.h"
 
 #define SPDK_ISCSI_CONNECTION_MEMSET(conn)		\
 	memset(&(conn)->portal, 0, sizeof(*(conn)) -	\
@@ -280,6 +279,7 @@ spdk_iscsi_conn_construct(struct spdk_iscsi_portal *portal,
 	TAILQ_INIT(&conn->queued_r2t_tasks);
 	TAILQ_INIT(&conn->active_r2t_tasks);
 	TAILQ_INIT(&conn->queued_datain_tasks);
+	memset(&conn->open_lun_descs, 0, sizeof(conn->open_lun_descs));
 
 	rc = spdk_sock_getaddr(sock, conn->target_addr,
 			       sizeof conn->target_addr,
@@ -583,14 +583,81 @@ spdk_iscsi_conn_check_shutdown(void *arg)
 
 	if (spdk_iscsi_get_active_conns() == 0) {
 		spdk_poller_unregister(&g_shutdown_timer);
-		event = spdk_event_allocate(spdk_env_get_current_core(), spdk_iscsi_conn_check_shutdown_cb, NULL,
+		event = spdk_event_allocate(spdk_env_get_current_core(),
-					    NULL);
+					    spdk_iscsi_conn_check_shutdown_cb, NULL, NULL);
 		spdk_event_call(event);
 	}
 
 	return -1;
 }
 
+static void
+spdk_iscsi_conn_close_lun(struct spdk_iscsi_conn *conn, int lun_id)
+{
+	struct spdk_scsi_desc *desc;
+
+	desc = conn->open_lun_descs[lun_id];
+	if (desc != NULL) {
+		spdk_scsi_lun_free_io_channel(desc);
+		spdk_scsi_lun_close(desc);
+		conn->open_lun_descs[lun_id] = NULL;
+	}
+}
+
+static void
+spdk_iscsi_conn_close_luns(struct spdk_iscsi_conn *conn)
+{
+	int i;
+
+	for (i = 0; i < SPDK_SCSI_DEV_MAX_LUN; i++) {
+		spdk_iscsi_conn_close_lun(conn, i);
+	}
+}
+
+static void
+spdk_iscsi_conn_remove_lun(struct spdk_scsi_lun *lun, void *remove_ctx)
+{
+	struct spdk_iscsi_conn *conn = remove_ctx;
+	int lun_id = spdk_scsi_lun_get_id(lun);
+
+	spdk_clear_all_transfer_task(conn, lun);
+
+	spdk_iscsi_conn_close_lun(conn, lun_id);
+}
+
+static void
+spdk_iscsi_conn_open_luns(struct spdk_iscsi_conn *conn)
+{
+	int i, rc;
+	struct spdk_scsi_lun *lun;
+	struct spdk_scsi_desc *desc;
+
+	for (i = 0; i < SPDK_SCSI_DEV_MAX_LUN; i++) {
+		lun = spdk_scsi_dev_get_lun(conn->dev, i);
+		if (lun == NULL) {
+			continue;
+		}
+
+		rc = spdk_scsi_lun_open(lun, spdk_iscsi_conn_remove_lun, conn, &desc);
+		if (rc != 0) {
+			goto error;
+		}
+
+		rc = spdk_scsi_lun_allocate_io_channel(desc);
+		if (rc != 0) {
+			spdk_scsi_lun_close(desc);
+			goto error;
+		}
+
+		conn->open_lun_descs[i] = desc;
+	}
+
+	return;
+
+error:
+	spdk_iscsi_conn_close_luns(conn);
+}
+
 /**
  *  This function will stop executing the specified connection.
  */
@@ -607,8 +674,7 @@ spdk_iscsi_conn_stop(struct spdk_iscsi_conn *conn)
 		target->num_active_conns--;
 		pthread_mutex_unlock(&target->mutex);
 
-		assert(conn->dev != NULL);
+		spdk_iscsi_conn_close_luns(conn);
-		spdk_scsi_dev_free_io_channels(conn->dev);
 	}
 
 	__sync_fetch_and_sub(&g_num_connections[spdk_env_get_current_core()], 1);
@@ -1171,8 +1237,7 @@ spdk_iscsi_conn_full_feature_migrate(void *arg1, void *arg2)
 	struct spdk_iscsi_conn *conn = arg1;
 
 	if (conn->sess->session_type == SESSION_TYPE_NORMAL) {
-		assert(conn->dev != NULL);
+		spdk_iscsi_conn_open_luns(conn);
-		spdk_scsi_dev_allocate_io_channels(conn->dev);
 	}
 
 	/* The poller has been unregistered, so now we can re-register it on the new core. */

lib/iscsi/conn.h

@@ -40,6 +40,7 @@
 #include "iscsi/iscsi.h"
 #include "spdk/queue.h"
 #include "spdk/cpuset.h"
+#include "spdk/scsi.h"
 
 /*
  * MAX_CONNECTION_PARAMS: The numbers of the params in conn_param_table
@@ -164,6 +165,8 @@ struct spdk_iscsi_conn {
 	TAILQ_HEAD(queued_r2t_tasks, spdk_iscsi_task)	queued_r2t_tasks;
 	TAILQ_HEAD(active_r2t_tasks, spdk_iscsi_task)	active_r2t_tasks;
 	TAILQ_HEAD(queued_datain_tasks, spdk_iscsi_task)	queued_datain_tasks;
+
+	struct spdk_scsi_desc	*open_lun_descs[SPDK_SCSI_DEV_MAX_LUN];
 };
 
 extern struct spdk_iscsi_conn *g_conns_array;

lib/iscsi/iscsi.c

@@ -3558,7 +3558,8 @@ void spdk_del_transfer_task(struct spdk_iscsi_conn *conn, uint32_t task_tag)
 }
 
 static void
-spdk_del_connection_queued_task(void *tailq, struct spdk_scsi_lun *lun)
+spdk_del_connection_queued_task(struct spdk_iscsi_conn *conn, void *tailq,
+				struct spdk_scsi_lun *lun)
 {
 	struct spdk_iscsi_task *task, *task_tmp;
 	/*
@@ -3571,6 +3572,10 @@ spdk_del_connection_queued_task(void *tailq, struct spdk_scsi_lun *lun)
 	TAILQ_FOREACH_SAFE(task, head, link, task_tmp) {
 		if (lun == NULL || lun == task->scsi.lun) {
 			TAILQ_REMOVE(head, task, link);
+			if (lun != NULL && spdk_scsi_lun_is_removing(lun)) {
+				spdk_scsi_task_process_null_lun(&task->scsi);
+				spdk_iscsi_task_response(conn, task);
+			}
 			spdk_iscsi_task_put(task);
 		}
 	}
@@ -3608,8 +3613,8 @@ void spdk_clear_all_transfer_task(struct spdk_iscsi_conn *conn,
 		}
 	}
 
-	spdk_del_connection_queued_task(&conn->active_r2t_tasks, lun);
+	spdk_del_connection_queued_task(conn, &conn->active_r2t_tasks, lun);
-	spdk_del_connection_queued_task(&conn->queued_r2t_tasks, lun);
+	spdk_del_connection_queued_task(conn, &conn->queued_r2t_tasks, lun);
 
 	spdk_start_queued_transfer_tasks(conn);
 }

lib/scsi/lun.c

@@ -444,3 +444,9 @@ spdk_scsi_lun_has_pending_tasks(const struct spdk_scsi_lun *lun)
 {
 	return !TAILQ_EMPTY(&lun->tasks);
 }
+
+bool
+spdk_scsi_lun_is_removing(const struct spdk_scsi_lun *lun)
+{
+	return lun->removed;
+}

test/unit/lib/iscsi/conn.c/conn_ut.c

@@ -157,13 +157,35 @@ spdk_scsi_task_put(struct spdk_scsi_task *task)
 {
 }
 
+struct spdk_scsi_lun *
+spdk_scsi_dev_get_lun(struct spdk_scsi_dev *dev, int lun_id)
+{
+	return NULL;
+}
+
+int
+spdk_scsi_lun_open(struct spdk_scsi_lun *lun, spdk_scsi_remove_cb_t hotremove_cb,
+		   void *hotremove_ctx, struct spdk_scsi_desc **desc)
+{
+	return 0;
+}
+
 void
-spdk_scsi_dev_free_io_channels(struct spdk_scsi_dev *dev)
+spdk_scsi_lun_close(struct spdk_scsi_desc *desc)
+{
+}
+
+int spdk_scsi_lun_allocate_io_channel(struct spdk_scsi_desc *desc)
+{
+	return 0;
+}
+
+void spdk_scsi_lun_free_io_channel(struct spdk_scsi_desc *desc)
 {
 }
 
 int
-spdk_scsi_dev_allocate_io_channels(struct spdk_scsi_dev *dev)
+spdk_scsi_lun_get_id(const struct spdk_scsi_lun *lun)
 {
 	return 0;
 }

test/unit/lib/iscsi/iscsi.c/iscsi_ut.c

@@ -102,6 +102,12 @@ spdk_scsi_lun_get_id(const struct spdk_scsi_lun *lun)
 	return lun->id;
 }
 
+bool
+spdk_scsi_lun_is_removing(const struct spdk_scsi_lun *lun)
+{
+	return true;
+}
+
 struct spdk_scsi_lun *
 spdk_scsi_dev_get_lun(struct spdk_scsi_dev *dev, int lun_id)
 {