From e6fea425c90a64598e066fba875c329f9f6e0a0f Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Wed, 22 Nov 2017 14:58:25 +0900 Subject: [PATCH] iscsi: Make MaxQueueDepth configurable by config file MaxQueueDepth is configurable for NVMf target by configuration file. This patch makes this flexibility possible for iSCSI target too. Increasing macro-defined constant for MaxQueueDepth consume extra memory regardless of the necessity and hence it may be better to avoid that. Change-Id: I7fa9cf86b02c5a8b16411d4b4d356c6e6c0dc3eb Signed-off-by: Shuhei Matsumoto Reviewed-on: https://review.gerrithub.io/388552 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Jim Harris --- lib/iscsi/iscsi.h | 7 ++++++- lib/iscsi/iscsi_subsystem.c | 20 +++++++++++++++++++- lib/iscsi/tgt_node.c | 14 +++++++------- lib/iscsi/tgt_node.h | 3 +-- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/lib/iscsi/iscsi.h b/lib/iscsi/iscsi.h index 37964999e..35c5628f5 100644 --- a/lib/iscsi/iscsi.h +++ b/lib/iscsi/iscsi.h @@ -107,7 +107,11 @@ */ #define MAX_LARGE_DATAIN_PER_CONNECTION 64 -#define NUM_PDU_PER_CONNECTION (2 * (SPDK_ISCSI_MAX_QUEUE_DEPTH + MAX_LARGE_DATAIN_PER_CONNECTION + 8)) +/* + * Defines default maximum queue depth per connection and this can be + * changed by configuration file. + */ +#define DEFAULT_MAX_QUEUE_DEPTH 64 #define SPDK_ISCSI_MAX_BURST_LENGTH \ (SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH * MAX_DATA_OUT_PER_CONNECTION) @@ -276,6 +280,7 @@ struct spdk_iscsi_globals { uint32_t MaxSessions; uint32_t MaxConnectionsPerSession; uint32_t MaxConnections; + uint32_t MaxQueueDepth; uint32_t DefaultTime2Wait; uint32_t DefaultTime2Retain; uint32_t ImmediateData; diff --git a/lib/iscsi/iscsi_subsystem.c b/lib/iscsi/iscsi_subsystem.c index 0771dc006..68c7ba941 100644 --- a/lib/iscsi/iscsi_subsystem.c +++ b/lib/iscsi/iscsi_subsystem.c @@ -71,6 +71,7 @@ static void *g_fini_cb_arg; " MaxSessions %d\n" \ " MaxConnectionsPerSession %d\n" \ " MaxConnections %d\n" \ +" MaxQueueDepth %d\n" \ "\n" \ " # iSCSI initial parameters negotiate with initiators\n" \ " # NOTE: incorrect values might crash\n" \ @@ -105,6 +106,7 @@ spdk_iscsi_config_dump_section(FILE *fp) g_spdk_iscsi.timeout, authmethod, authgroup, g_spdk_iscsi.MaxSessions, g_spdk_iscsi.MaxConnectionsPerSession, g_spdk_iscsi.MaxConnections, + g_spdk_iscsi.MaxQueueDepth, g_spdk_iscsi.DefaultTime2Wait, g_spdk_iscsi.DefaultTime2Retain, (g_spdk_iscsi.ImmediateData == 1) ? "Yes" : "No", g_spdk_iscsi.ErrorRecoveryLevel); @@ -320,7 +322,8 @@ spdk_mobj_ctor(struct rte_mempool *mp, __attribute__((unused)) void *arg, #endif } -#define PDU_POOL_SIZE(iscsi) (iscsi->MaxConnections * NUM_PDU_PER_CONNECTION) +#define NUM_PDU_PER_CONNECTION(iscsi) (2 * (iscsi->MaxQueueDepth + MAX_LARGE_DATAIN_PER_CONNECTION + 8)) +#define PDU_POOL_SIZE(iscsi) (iscsi->MaxConnections * NUM_PDU_PER_CONNECTION(iscsi)) #define IMMEDIATE_DATA_POOL_SIZE(iscsi) (iscsi->MaxConnections * 128) #define DATA_OUT_POOL_SIZE(iscsi) (iscsi->MaxConnections * MAX_DATA_OUT_PER_CONNECTION) @@ -533,6 +536,7 @@ spdk_iscsi_log_globals(void) SPDK_DEBUGLOG(SPDK_TRACE_ISCSI, "MaxSessions %d\n", g_spdk_iscsi.MaxSessions); SPDK_DEBUGLOG(SPDK_TRACE_ISCSI, "MaxConnectionsPerSession %d\n", g_spdk_iscsi.MaxConnectionsPerSession); + SPDK_DEBUGLOG(SPDK_TRACE_ISCSI, "MaxQueueDepth %d\n", g_spdk_iscsi.MaxQueueDepth); SPDK_DEBUGLOG(SPDK_TRACE_ISCSI, "DefaultTime2Wait %d\n", g_spdk_iscsi.DefaultTime2Wait); SPDK_DEBUGLOG(SPDK_TRACE_ISCSI, "DefaultTime2Retain %d\n", @@ -577,6 +581,7 @@ spdk_iscsi_read_parameters_from_config_file(struct spdk_conf_section *sp) char *authfile, *nodebase; int MaxSessions; int MaxConnectionsPerSession; + int MaxQueueDepth; int DefaultTime2Wait; int DefaultTime2Retain; int ErrorRecoveryLevel; @@ -639,6 +644,18 @@ spdk_iscsi_read_parameters_from_config_file(struct spdk_conf_section *sp) g_spdk_iscsi.MaxConnectionsPerSession = MaxConnectionsPerSession; } } + + MaxQueueDepth = spdk_conf_section_get_intval(sp, "MaxQueueDepth"); + if (MaxQueueDepth >= 0) { + if (MaxQueueDepth == 0) { + SPDK_ERRLOG("MaxQueueDepth == 0 invalid, ignoring\n"); + } else if (MaxQueueDepth > 256) { + SPDK_ERRLOG("MaxQueueDepth == %d invalid, ignoring\n", MaxQueueDepth); + } else { + g_spdk_iscsi.MaxQueueDepth = MaxQueueDepth; + } + } + DefaultTime2Wait = spdk_conf_section_get_intval(sp, "DefaultTime2Wait"); if (DefaultTime2Wait >= 0) { if (DefaultTime2Wait > 3600) { @@ -745,6 +762,7 @@ spdk_iscsi_app_read_parameters(void) g_spdk_iscsi.MaxSessions = DEFAULT_MAX_SESSIONS; g_spdk_iscsi.MaxConnectionsPerSession = DEFAULT_MAX_CONNECTIONS_PER_SESSION; + g_spdk_iscsi.MaxQueueDepth = DEFAULT_MAX_QUEUE_DEPTH; g_spdk_iscsi.DefaultTime2Wait = DEFAULT_DEFAULTTIME2WAIT; g_spdk_iscsi.DefaultTime2Retain = DEFAULT_DEFAULTTIME2RETAIN; g_spdk_iscsi.ImmediateData = DEFAULT_IMMEDIATEDATA; diff --git a/lib/iscsi/tgt_node.c b/lib/iscsi/tgt_node.c index ae52ae00e..ed5304472 100644 --- a/lib/iscsi/tgt_node.c +++ b/lib/iscsi/tgt_node.c @@ -829,13 +829,13 @@ spdk_iscsi_tgt_node_construct(int target_index, target->header_digest = header_digest; target->data_digest = data_digest; - if (queue_depth > SPDK_ISCSI_MAX_QUEUE_DEPTH) { - SPDK_DEBUGLOG(SPDK_TRACE_ISCSI, "QueueDepth %d > Max %d. Using %d instead.\n", - queue_depth, SPDK_ISCSI_MAX_QUEUE_DEPTH, - SPDK_ISCSI_MAX_QUEUE_DEPTH); - queue_depth = SPDK_ISCSI_MAX_QUEUE_DEPTH; + if (queue_depth > 0 && ((uint32_t)queue_depth <= g_spdk_iscsi.MaxQueueDepth)) { + target->queue_depth = queue_depth; + } else { + SPDK_DEBUGLOG(SPDK_TRACE_ISCSI, "QueueDepth %d is invalid and %d is used instead.\n", + queue_depth, g_spdk_iscsi.MaxQueueDepth); + target->queue_depth = g_spdk_iscsi.MaxQueueDepth; } - target->queue_depth = queue_depth; rc = spdk_iscsi_tgt_node_register(target); if (rc != 0) { @@ -1019,7 +1019,7 @@ spdk_cf_add_iscsi_tgt_node(struct spdk_conf_section *sp) val = spdk_conf_section_get_val(sp, "QueueDepth"); if (val == NULL) { - queue_depth = SPDK_ISCSI_MAX_QUEUE_DEPTH; + queue_depth = g_spdk_iscsi.MaxQueueDepth; } else { queue_depth = (int) strtol(val, NULL, 10); } diff --git a/lib/iscsi/tgt_node.h b/lib/iscsi/tgt_node.h index e232d3084..084cf3426 100644 --- a/lib/iscsi/tgt_node.h +++ b/lib/iscsi/tgt_node.h @@ -44,9 +44,8 @@ struct spdk_iscsi_init_grp; struct spdk_iscsi_portal_grp; struct spdk_iscsi_portal; -#define SPDK_ISCSI_MAX_QUEUE_DEPTH 64 #define MAX_TARGET_MAP 256 -#define SPDK_TN_TAG_MAX 0x0000ffff +#define SPDK_TN_TAG_MAX 0x0000ffff struct spdk_iscsi_ig_map { struct spdk_iscsi_init_grp *ig;