From f6823c5907515e080d4938a81488a1042d51ceb5 Mon Sep 17 00:00:00 2001 From: Seth Howell Date: Fri, 26 Apr 2019 14:25:20 -0700 Subject: [PATCH] rdma: add a flag for disabling srq. There are cases where srq can be a detriment. Add a flag to allow users to disable srq even if they have a piece of hardware that supports it. Signed-off-by: Seth Howell Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/452271 (master) (cherry picked from commit 350e429a57b0f2beb3fe71560a6dc34aa220cc58) Change-Id: Ia3be8e8c8e8463964e6ff1c02b07afbf4c3cc8f7 Signed-off-by: Tomasz Zawadzki Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/457195 Tested-by: SPDK CI Jenkins Reviewed-by: Darek Stojaczyk --- CHANGELOG.md | 6 ++++++ doc/jsonrpc.md | 1 + include/spdk/nvmf.h | 19 ++++++++++--------- lib/event/subsystems/nvmf/nvmf_rpc.c | 4 ++++ lib/nvmf/rdma.c | 9 ++++++--- scripts/rpc.py | 4 +++- scripts/rpc/nvmf.py | 6 +++++- 7 files changed, 35 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d25e1f0..690470a42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## v19.04.1: (Upcoming Release) +### NVMe-oF Target + +Shared receive queue can now be disabled even for NICs that support it using the +`nvmf_create_transport` RPC method parameter `no_srq`. The actual use of a shared +receive queue is predicated on hardware support when this flag is not used. + ## v19.04: ### nvme diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 9103a0fa1..7ee6ce9f1 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -3488,6 +3488,7 @@ max_aq_depth | Optional | number | Max number of admin cmds per num_shared_buffers | Optional | number | The number of pooled data buffers available to the transport buf_cache_size | Optional | number | The number of shared buffers to reserve for each poll group max_srq_depth | Optional | number | The number of elements in a per-thread shared receive queue (RDMA only) +no_srq | Optional | boolean | Disable shared receive queue even for devices that support it. (RDMA only) ### Example: diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index 0c92880fe..3cf7e47ea 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -64,15 +64,16 @@ struct spdk_json_write_ctx; struct spdk_nvmf_transport; struct spdk_nvmf_transport_opts { - uint16_t max_queue_depth; - uint16_t max_qpairs_per_ctrlr; - uint32_t in_capsule_data_size; - uint32_t max_io_size; - uint32_t io_unit_size; - uint32_t max_aq_depth; - uint32_t num_shared_buffers; - uint32_t buf_cache_size; - uint32_t max_srq_depth; + uint16_t max_queue_depth; + uint16_t max_qpairs_per_ctrlr; + uint32_t in_capsule_data_size; + uint32_t max_io_size; + uint32_t io_unit_size; + uint32_t max_aq_depth; + uint32_t num_shared_buffers; + uint32_t buf_cache_size; + uint32_t max_srq_depth; + bool no_srq; }; /** diff --git a/lib/event/subsystems/nvmf/nvmf_rpc.c b/lib/event/subsystems/nvmf/nvmf_rpc.c index 3a19aad5e..2c9dae5b9 100644 --- a/lib/event/subsystems/nvmf/nvmf_rpc.c +++ b/lib/event/subsystems/nvmf/nvmf_rpc.c @@ -1452,6 +1452,10 @@ static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[] "max_srq_depth", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_srq_depth), spdk_json_decode_uint32, true }, + { + "no_srq", offsetof(struct nvmf_rpc_create_transport_ctx, opts.no_srq), + spdk_json_decode_bool, true + }, }; static void diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index c4a6cdb73..cdb3300a3 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -2047,6 +2047,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, #define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES) #define SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS 4096 #define SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE 32 +#define SPDK_NVMF_RDMA_DEFAULT_NO_SRQ false; static void spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) @@ -2060,6 +2061,7 @@ spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) opts->num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS; opts->buf_cache_size = SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE; opts->max_srq_depth = SPDK_NVMF_RDMA_DEFAULT_SRQ_DEPTH; + opts->no_srq = SPDK_NVMF_RDMA_DEFAULT_NO_SRQ } const struct spdk_mem_map_ops g_nvmf_rdma_map_ops = { @@ -2103,7 +2105,7 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) " Transport opts: max_ioq_depth=%d, max_io_size=%d,\n" " max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n" " in_capsule_data_size=%d, max_aq_depth=%d,\n" - " num_shared_buffers=%d, max_srq_depth=%d\n", + " num_shared_buffers=%d, max_srq_depth=%d, no_srq=%d\n", opts->max_queue_depth, opts->max_io_size, opts->max_qpairs_per_ctrlr, @@ -2111,7 +2113,8 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) opts->in_capsule_data_size, opts->max_aq_depth, opts->num_shared_buffers, - opts->max_srq_depth); + opts->max_srq_depth, + opts->no_srq); /* I/O unit size cannot be larger than max I/O size */ if (opts->io_unit_size > opts->max_io_size) { @@ -2940,7 +2943,7 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) TAILQ_INIT(&poller->qpairs); TAILQ_INSERT_TAIL(&rgroup->pollers, poller, link); - if (device->attr.max_srq != 0) { + if (transport->opts.no_srq == false && device->attr.max_srq != 0) { poller->max_srq_depth = transport->opts.max_srq_depth; memset(&srq_init_attr, 0, sizeof(struct ibv_srq_init_attr)); diff --git a/scripts/rpc.py b/scripts/rpc.py index 02f6e83c7..6a1d0a8e2 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1403,7 +1403,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse max_aq_depth=args.max_aq_depth, num_shared_buffers=args.num_shared_buffers, buf_cache_size=args.buf_cache_size, - max_srq_depth=args.max_srq_depth) + max_srq_depth=args.max_srq_depth, + no_srq=args.no_srq) p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport') p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True) @@ -1416,6 +1417,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p.add_argument('-n', '--num-shared-buffers', help='The number of pooled data buffers available to the transport', type=int) p.add_argument('-b', '--buf-cache-size', help='The number of shared buffers to reserve for each poll group', type=int) p.add_argument('-s', '--max-srq-depth', help='Max number of outstanding I/O per SRQ. Relevant only for RDMA transport', type=int) + p.add_argument('-r', '--no-srq', action='store_true', help='Disable per-thread shared receive queue. Relevant only for RDMA transport') p.set_defaults(func=nvmf_create_transport) def get_nvmf_transports(args): diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index d800e893c..7f0cebcad 100644 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -45,7 +45,8 @@ def nvmf_create_transport(client, max_aq_depth=None, num_shared_buffers=None, buf_cache_size=None, - max_srq_depth=None): + max_srq_depth=None, + no_srq=False): """NVMf Transport Create options. Args: @@ -59,6 +60,7 @@ def nvmf_create_transport(client, num_shared_buffers: The number of pooled data buffers available to the transport (optional) buf_cache_size: The number of shared buffers to reserve for each poll group (optional) max_srq_depth: Max number of outstanding I/O per shared receive queue - RDMA specific (optional) + no_srq: Boolean flag to disable SRQ even for devices that support it - RDMA specific (optional) Returns: True or False @@ -84,6 +86,8 @@ def nvmf_create_transport(client, params['buf_cache_size'] = buf_cache_size if max_srq_depth: params['max_srq_depth'] = max_srq_depth + if no_srq: + params['no_srq'] = no_srq return client.call('nvmf_create_transport', params)