From d393983d74c9a0c7548a4eeae475210a7fc01742 Mon Sep 17 00:00:00 2001 From: GangCao Date: Fri, 10 Aug 2018 14:27:02 -0400 Subject: [PATCH] lib/iscsi: export FirstBurstLength as user configurable parameter According to https://tools.ietf.org/html/rfc3720, the default value for the FirstBurstLength is 65536 bytes while SPDK iSCSI target picks the smaller 8192 as the default setting. This value is the communication for the iSCSI initiator to send the unsolicited data and instead of having a fixed setting here, expose it as a user configurable parameter to fit the real use case, especially for the data out iSCSI write. Example of usage as following in the iSCSI.conf: FirstBurstLength 8192 Change-Id: I71690c7c48aa0875f1f975c0ea935389de6d1e6d Signed-off-by: GangCao Reviewed-on: https://review.gerrithub.io/421142 Chandler-Test-Pool: SPDK Automated Test System Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- doc/jsonrpc.md | 3 +++ etc/spdk/iscsi.conf.in | 5 ++++ lib/event/subsystems/iscsi/iscsi_rpc.c | 1 + lib/iscsi/iscsi.c | 6 ++--- lib/iscsi/iscsi.h | 17 ++++++++++++- lib/iscsi/iscsi_subsystem.c | 33 ++++++++++++++++++++++++-- scripts/rpc.py | 2 ++ scripts/rpc/iscsi.py | 4 ++++ 8 files changed, 65 insertions(+), 6 deletions(-) diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index e5e70439e..84dc27c6d 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -1784,6 +1784,7 @@ max_queue_depth | number | Maximum number of outstanding I/Os per q max_connections_per_session | number | Session specific parameter, MaxConnections (default: 2) default_time2wait | number | Session specific parameter, DefaultTime2Wait (default: 2) default_time2retain | number | Session specific parameter, DefaultTime2Retain (default: 20) +first_burst_length | number | Session specific parameter, FirstBurstLength (default: 8192) immediate_data | boolean | Session specific parameter, ImmediateData (default: `true`) error_recovery_level | number | Session specific parameter, ErrorRecoveryLevel (default: 0) allow_duplicated_isid | boolean | Allow duplicated initiator session ID (default: `false`) @@ -1800,6 +1801,7 @@ Example request: "params": { "allow_duplicated_isid": true, "default_time2retain": 60, + "first_burst_length": 8192, "immediate_data": true, "node_base": "iqn.2016-06.io.spdk", "max_sessions": 128, @@ -1855,6 +1857,7 @@ Example response: "result": { "allow_duplicated_isid": true, "default_time2retain": 60, + "first_burst_length": 8192, "immediate_data": true, "node_base": "iqn.2016-06.io.spdk", "mutual_chap": false, diff --git a/etc/spdk/iscsi.conf.in b/etc/spdk/iscsi.conf.in index 1ebee8ed4..5171190d8 100644 --- a/etc/spdk/iscsi.conf.in +++ b/etc/spdk/iscsi.conf.in @@ -63,6 +63,11 @@ DefaultTime2Wait 2 DefaultTime2Retain 60 + # Maximum amount in bytes of unsolicited data the iSCSI + # initiator may send to the target during the execution of + # a single SCSI command. + FirstBurstLength 8192 + ImmediateData Yes ErrorRecoveryLevel 0 diff --git a/lib/event/subsystems/iscsi/iscsi_rpc.c b/lib/event/subsystems/iscsi/iscsi_rpc.c index 23233520e..49c3de0e5 100644 --- a/lib/event/subsystems/iscsi/iscsi_rpc.c +++ b/lib/event/subsystems/iscsi/iscsi_rpc.c @@ -58,6 +58,7 @@ static const struct spdk_json_object_decoder rpc_set_iscsi_opts_decoders[] = { {"max_connections_per_session", offsetof(struct spdk_iscsi_opts, MaxConnectionsPerSession), spdk_json_decode_uint32, true}, {"default_time2wait", offsetof(struct spdk_iscsi_opts, DefaultTime2Wait), spdk_json_decode_uint32, true}, {"default_time2retain", offsetof(struct spdk_iscsi_opts, DefaultTime2Retain), spdk_json_decode_uint32, true}, + {"first_burst_length", offsetof(struct spdk_iscsi_opts, FirstBurstLength), spdk_json_decode_uint32, true}, {"immediate_data", offsetof(struct spdk_iscsi_opts, ImmediateData), spdk_json_decode_bool, true}, {"error_recovery_level", offsetof(struct spdk_iscsi_opts, ErrorRecoveryLevel), spdk_json_decode_uint32, true}, {"allow_duplicated_isid", offsetof(struct spdk_iscsi_opts, AllowDuplicateIsid), spdk_json_decode_bool, true}, diff --git a/lib/iscsi/iscsi.c b/lib/iscsi/iscsi.c index 6e70bc47f..08c0f5544 100644 --- a/lib/iscsi/iscsi.c +++ b/lib/iscsi/iscsi.c @@ -1028,9 +1028,9 @@ spdk_iscsi_check_values(struct spdk_iscsi_conn *conn) conn->sess->MaxBurstLength); return -1; } - if (conn->sess->FirstBurstLength > SPDK_ISCSI_FIRST_BURST_LENGTH) { + if (conn->sess->FirstBurstLength > g_spdk_iscsi.FirstBurstLength) { SPDK_ERRLOG("FirstBurstLength(%d) > iSCSI target restriction(%d)\n", - conn->sess->FirstBurstLength, SPDK_ISCSI_FIRST_BURST_LENGTH); + conn->sess->FirstBurstLength, g_spdk_iscsi.FirstBurstLength); return -1; } if (conn->sess->MaxBurstLength > 0x00ffffff) { @@ -4384,7 +4384,7 @@ spdk_create_iscsi_sess(struct spdk_iscsi_conn *conn, sess->DefaultTime2Wait = g_spdk_iscsi.DefaultTime2Wait; sess->DefaultTime2Retain = g_spdk_iscsi.DefaultTime2Retain; - sess->FirstBurstLength = SPDK_ISCSI_FIRST_BURST_LENGTH; + sess->FirstBurstLength = g_spdk_iscsi.FirstBurstLength; sess->MaxBurstLength = SPDK_ISCSI_MAX_BURST_LENGTH; sess->InitialR2T = DEFAULT_INITIALR2T; sess->ImmediateData = g_spdk_iscsi.ImmediateData; diff --git a/lib/iscsi/iscsi.h b/lib/iscsi/iscsi.h index 7a72b0a29..249aa866b 100644 --- a/lib/iscsi/iscsi.h +++ b/lib/iscsi/iscsi.h @@ -46,6 +46,7 @@ #include "iscsi/tgt_node.h" #include "spdk/assert.h" +#include "spdk/util.h" #define SPDK_ISCSI_BUILD_ETC "/usr/local/etc/spdk" #define SPDK_ISCSI_DEFAULT_CONFIG SPDK_ISCSI_BUILD_ETC "/iscsi.conf" @@ -114,8 +115,20 @@ #define SPDK_ISCSI_MAX_BURST_LENGTH \ (SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH * MAX_DATA_OUT_PER_CONNECTION) +/* + * Defines default maximum amount in bytes of unsolicited data the iSCSI + * initiator may send to the SPDK iSCSI target during the execution of + * a single SCSI command. And it is smaller than the MaxBurstLength. + */ #define SPDK_ISCSI_FIRST_BURST_LENGTH 8192 +/* + * Defines minimum amount in bytes of unsolicited data the iSCSI initiator + * may send to the SPDK iSCSI target during the execution of a single + * SCSI command. + */ +#define SPDK_ISCSI_MIN_FIRST_BURST_LENGTH 512 + /** Defines how long we should wait for a TCP close after responding to a * logout request, before terminating the connection ourselves. */ @@ -280,6 +293,7 @@ struct spdk_iscsi_opts { uint32_t MaxQueueDepth; uint32_t DefaultTime2Wait; uint32_t DefaultTime2Retain; + uint32_t FirstBurstLength; bool ImmediateData; uint32_t ErrorRecoveryLevel; bool AllowDuplicateIsid; @@ -308,6 +322,7 @@ struct spdk_iscsi_globals { uint32_t MaxQueueDepth; uint32_t DefaultTime2Wait; uint32_t DefaultTime2Retain; + uint32_t FirstBurstLength; bool ImmediateData; uint32_t ErrorRecoveryLevel; bool AllowDuplicateIsid; @@ -412,7 +427,7 @@ spdk_get_immediate_data_buffer_size(void) * take up much space and we need to make sure the worst-case scenario * can be satisified by the size returned here. */ - return SPDK_ISCSI_FIRST_BURST_LENGTH + + return g_spdk_iscsi.FirstBurstLength + ISCSI_DIGEST_LEN + /* data digest */ ISCSI_DIGEST_LEN + /* header digest */ 8 + /* bidirectional AHS */ diff --git a/lib/iscsi/iscsi_subsystem.c b/lib/iscsi/iscsi_subsystem.c index f1264807e..91c2b4d44 100644 --- a/lib/iscsi/iscsi_subsystem.c +++ b/lib/iscsi/iscsi_subsystem.c @@ -81,6 +81,7 @@ static void *g_fini_cb_arg; " DefaultTime2Wait %d\n" \ " DefaultTime2Retain %d\n" \ "\n" \ +" FirstBurstLength %d\n" \ " ImmediateData %s\n" \ " ErrorRecoveryLevel %d\n" \ "\n" @@ -114,6 +115,7 @@ spdk_iscsi_globals_config_text(FILE *fp) g_spdk_iscsi.MaxConnections, g_spdk_iscsi.MaxQueueDepth, g_spdk_iscsi.DefaultTime2Wait, g_spdk_iscsi.DefaultTime2Retain, + g_spdk_iscsi.FirstBurstLength, (g_spdk_iscsi.ImmediateData) ? "Yes" : "No", g_spdk_iscsi.ErrorRecoveryLevel); } @@ -168,7 +170,7 @@ static int spdk_iscsi_initialize_pdu_pool(void) spdk_env_get_socket_id(spdk_env_get_current_core()), spdk_mobj_ctor, NULL); if (!iscsi->pdu_immediate_data_pool) { - SPDK_ERRLOG("create PDU 8k pool failed\n"); + SPDK_ERRLOG("create PDU immediate data pool failed\n"); return -1; } @@ -178,7 +180,7 @@ static int spdk_iscsi_initialize_pdu_pool(void) spdk_env_get_socket_id(spdk_env_get_current_core()), spdk_mobj_ctor, NULL); if (!iscsi->pdu_data_out_pool) { - SPDK_ERRLOG("create PDU 64k pool failed\n"); + SPDK_ERRLOG("create PDU data out pool failed\n"); return -1; } @@ -342,6 +344,8 @@ spdk_iscsi_log_globals(void) g_spdk_iscsi.DefaultTime2Wait); SPDK_DEBUGLOG(SPDK_LOG_ISCSI, "DefaultTime2Retain %d\n", g_spdk_iscsi.DefaultTime2Retain); + SPDK_DEBUGLOG(SPDK_LOG_ISCSI, "FirstBurstLength %d\n", + g_spdk_iscsi.FirstBurstLength); SPDK_DEBUGLOG(SPDK_LOG_ISCSI, "ImmediateData %s\n", g_spdk_iscsi.ImmediateData ? "Yes" : "No"); SPDK_DEBUGLOG(SPDK_LOG_ISCSI, "AllowDuplicateIsid %s\n", @@ -385,6 +389,7 @@ spdk_iscsi_opts_init(struct spdk_iscsi_opts *opts) opts->MaxQueueDepth = DEFAULT_MAX_QUEUE_DEPTH; opts->DefaultTime2Wait = DEFAULT_DEFAULTTIME2WAIT; opts->DefaultTime2Retain = DEFAULT_DEFAULTTIME2RETAIN; + opts->FirstBurstLength = DEFAULT_FIRSTBURSTLENGTH; opts->ImmediateData = DEFAULT_IMMEDIATEDATA; opts->AllowDuplicateIsid = false; opts->ErrorRecoveryLevel = DEFAULT_ERRORRECOVERYLEVEL; @@ -459,6 +464,7 @@ spdk_iscsi_opts_copy(struct spdk_iscsi_opts *src) dst->MaxQueueDepth = src->MaxQueueDepth; dst->DefaultTime2Wait = src->DefaultTime2Wait; dst->DefaultTime2Retain = src->DefaultTime2Retain; + dst->FirstBurstLength = src->FirstBurstLength; dst->ImmediateData = src->ImmediateData; dst->AllowDuplicateIsid = src->AllowDuplicateIsid; dst->ErrorRecoveryLevel = src->ErrorRecoveryLevel; @@ -483,6 +489,7 @@ spdk_iscsi_read_config_file_params(struct spdk_conf_section *sp, int MaxQueueDepth; int DefaultTime2Wait; int DefaultTime2Retain; + int FirstBurstLength; int ErrorRecoveryLevel; int timeout; int nopininterval; @@ -533,10 +540,17 @@ spdk_iscsi_read_config_file_params(struct spdk_conf_section *sp, if (DefaultTime2Wait >= 0) { opts->DefaultTime2Wait = DefaultTime2Wait; } + DefaultTime2Retain = spdk_conf_section_get_intval(sp, "DefaultTime2Retain"); if (DefaultTime2Retain >= 0) { opts->DefaultTime2Retain = DefaultTime2Retain; } + + FirstBurstLength = spdk_conf_section_get_intval(sp, "FirstBurstLength"); + if (FirstBurstLength >= 0) { + opts->FirstBurstLength = FirstBurstLength; + } + opts->ImmediateData = spdk_conf_section_get_boolval(sp, "ImmediateData", opts->ImmediateData); @@ -654,6 +668,18 @@ spdk_iscsi_opts_verify(struct spdk_iscsi_opts *opts) return -EINVAL; } + if (opts->FirstBurstLength >= SPDK_ISCSI_MIN_FIRST_BURST_LENGTH) { + if (opts->FirstBurstLength > SPDK_ISCSI_MAX_BURST_LENGTH) { + SPDK_ERRLOG("FirstBurstLength %d shall not exceed MaxBurstLength %d\n", + opts->FirstBurstLength, SPDK_ISCSI_MAX_BURST_LENGTH); + return -EINVAL; + } + } else { + SPDK_ERRLOG("FirstBurstLength %d shall be no less than %d\n", + opts->FirstBurstLength, SPDK_ISCSI_MIN_FIRST_BURST_LENGTH); + return -EINVAL; + } + if (opts->ErrorRecoveryLevel > 2) { SPDK_ERRLOG("ErrorRecoveryLevel %d is not supported.\n", opts->ErrorRecoveryLevel); return -EINVAL; @@ -737,6 +763,7 @@ spdk_iscsi_set_global_params(struct spdk_iscsi_opts *opts) g_spdk_iscsi.MaxQueueDepth = opts->MaxQueueDepth; g_spdk_iscsi.DefaultTime2Wait = opts->DefaultTime2Wait; g_spdk_iscsi.DefaultTime2Retain = opts->DefaultTime2Retain; + g_spdk_iscsi.FirstBurstLength = opts->FirstBurstLength; g_spdk_iscsi.ImmediateData = opts->ImmediateData; g_spdk_iscsi.AllowDuplicateIsid = opts->AllowDuplicateIsid; g_spdk_iscsi.ErrorRecoveryLevel = opts->ErrorRecoveryLevel; @@ -1135,6 +1162,8 @@ spdk_iscsi_opts_info_json(struct spdk_json_write_ctx *w) spdk_json_write_named_uint32(w, "default_time2wait", g_spdk_iscsi.DefaultTime2Wait); spdk_json_write_named_uint32(w, "default_time2retain", g_spdk_iscsi.DefaultTime2Retain); + spdk_json_write_named_uint32(w, "first_burst_length", g_spdk_iscsi.FirstBurstLength); + spdk_json_write_named_bool(w, "immediate_data", g_spdk_iscsi.ImmediateData); spdk_json_write_named_bool(w, "allow_duplicated_isid", g_spdk_iscsi.AllowDuplicateIsid); diff --git a/scripts/rpc.py b/scripts/rpc.py index 44d80c0b0..c331e8225 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -502,6 +502,7 @@ if __name__ == "__main__": max_connections_per_session=args.max_connections_per_session, default_time2wait=args.default_time2wait, default_time2retain=args.default_time2retain, + first_burst_length=args.first_burst_length, immediate_data=args.immediate_data, error_recovery_level=args.error_recovery_level, allow_duplicated_isid=args.allow_duplicated_isid, @@ -524,6 +525,7 @@ if __name__ == "__main__": p.add_argument('-c', '--max-connections-per-session', help='Negotiated parameter, MaxConnections.', type=int) p.add_argument('-w', '--default-time2wait', help='Negotiated parameter, DefaultTime2Wait.', type=int) p.add_argument('-v', '--default-time2retain', help='Negotiated parameter, DefaultTime2Retain.', type=int) + p.add_argument('-s', '--first-burst-length', help='Negotiated parameter, FirstBurstLength.', type=int) p.add_argument('-i', '--immediate-data', help='Negotiated parameter, ImmediateData.', action='store_true') p.add_argument('-l', '--error-recovery-level', help='Negotiated parameter, ErrorRecoveryLevel', type=int) p.add_argument('-p', '--allow-duplicated-isid', help='Allow duplicated initiator session ID.', action='store_true') diff --git a/scripts/rpc/iscsi.py b/scripts/rpc/iscsi.py index 4d616ffc3..b4042671a 100755 --- a/scripts/rpc/iscsi.py +++ b/scripts/rpc/iscsi.py @@ -15,6 +15,7 @@ def set_iscsi_options( max_connections_per_session=None, default_time2wait=None, default_time2retain=None, + first_burst_length=None, immediate_data=None, error_recovery_level=None, allow_duplicated_isid=None, @@ -35,6 +36,7 @@ def set_iscsi_options( max_connections_per_session: Negotiated parameter, MaxConnections default_time2wait: Negotiated parameter, DefaultTime2Wait default_time2retain: Negotiated parameter, DefaultTime2Retain + first_burst_length: Negotiated parameter, FirstBurstLength immediate_data: Negotiated parameter, ImmediateData error_recovery_level: Negotiated parameter, ErrorRecoveryLevel allow_duplicated_isid: Allow duplicated initiator session ID @@ -71,6 +73,8 @@ def set_iscsi_options( params['default_time2wait'] = default_time2wait if default_time2retain: params['default_time2retain'] = default_time2retain + if first_burst_length: + params['first_burst_length'] = first_burst_length if immediate_data: params['immediate_data'] = immediate_data if error_recovery_level: