diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 376ba2ca3..803864363 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -3617,6 +3617,7 @@ disable_auto_failback | Optional | boolean | Disable automatic failback generate_uuids | Optional | boolean | Enable generation of UUIDs for NVMe bdevs that do not provide this value themselves. transport_tos | Optional | number | IPv4 Type of Service value. Only applicable for RDMA transport. Default: 0 (no TOS is applied). nvme_error_stat | Optional | boolean | Enable collecting NVMe error counts. +rdma_srq_size | Optional | number | Set the size of a shared rdma receive queue. Default: 0 (disabled). #### Example diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index 28a7c2446..ed923064d 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -4621,7 +4621,9 @@ bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) int bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) { - int ret = bdev_nvme_validate_opts(opts); + int ret; + + ret = bdev_nvme_validate_opts(opts); if (ret) { SPDK_WARNLOG("Failed to set nvme opts.\n"); return ret; @@ -4633,6 +4635,19 @@ bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) } } + if (opts->rdma_srq_size != 0) { + struct spdk_nvme_transport_opts drv_opts; + + spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts)); + drv_opts.rdma_srq_size = opts->rdma_srq_size; + + ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts)); + if (ret) { + SPDK_ERRLOG("Failed to set NVMe transport opts.\n"); + return ret; + } + } + g_opts = *opts; return 0; diff --git a/module/bdev/nvme/bdev_nvme.h b/module/bdev/nvme/bdev_nvme.h index 81b877e9b..2a80e00ce 100644 --- a/module/bdev/nvme/bdev_nvme.h +++ b/module/bdev/nvme/bdev_nvme.h @@ -263,6 +263,7 @@ struct spdk_bdev_nvme_opts { /* Type of Service - RDMA only */ uint8_t transport_tos; bool nvme_error_stat; + uint32_t rdma_srq_size; }; struct spdk_nvme_qpair *bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch); diff --git a/module/bdev/nvme/bdev_nvme_rpc.c b/module/bdev/nvme/bdev_nvme_rpc.c index 04463a4ec..a563ac122 100644 --- a/module/bdev/nvme/bdev_nvme_rpc.c +++ b/module/bdev/nvme/bdev_nvme_rpc.c @@ -72,6 +72,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_options_decoders[] = {"generate_uuids", offsetof(struct spdk_bdev_nvme_opts, generate_uuids), spdk_json_decode_bool, true}, {"transport_tos", offsetof(struct spdk_bdev_nvme_opts, transport_tos), spdk_json_decode_uint8, true}, {"nvme_error_stat", offsetof(struct spdk_bdev_nvme_opts, nvme_error_stat), spdk_json_decode_bool, true}, + {"rdma_srq_size", offsetof(struct spdk_bdev_nvme_opts, rdma_srq_size), spdk_json_decode_uint32, true}, }; static void diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index e754a1465..9d7cbe435 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -533,7 +533,7 @@ def bdev_nvme_set_options(client, action_on_timeout=None, timeout_us=None, timeo delay_cmd_submit=None, transport_retry_count=None, bdev_retry_count=None, transport_ack_timeout=None, ctrlr_loss_timeout_sec=None, reconnect_delay_sec=None, fast_io_fail_timeout_sec=None, disable_auto_failback=None, generate_uuids=None, - transport_tos=None, nvme_error_stat=None): + transport_tos=None, nvme_error_stat=None, rdma_srq_size=None): """Set options for the bdev nvme. This is startup command. Args: @@ -576,6 +576,7 @@ def bdev_nvme_set_options(client, action_on_timeout=None, timeout_us=None, timeo transport_tos: IPv4 Type of Service value. Only applicable for RDMA transports. The default is 0 which means no TOS is applied. (optional) nvme_error_stat: Enable collecting NVMe error counts. (optional) + rdma_srq_size: Set the size of a shared rdma receive queue. Default: 0 (disabled) (optional) """ params = {} @@ -650,6 +651,9 @@ def bdev_nvme_set_options(client, action_on_timeout=None, timeout_us=None, timeo if nvme_error_stat is not None: params['nvme_error_stat'] = nvme_error_stat + if rdma_srq_size is not None: + params['rdma_srq_size'] = rdma_srq_size + return client.call('bdev_nvme_set_options', params) diff --git a/scripts/rpc.py b/scripts/rpc.py index 83780031c..acafef1da 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -561,7 +561,8 @@ if __name__ == "__main__": disable_auto_failback=args.disable_auto_failback, generate_uuids=args.generate_uuids, transport_tos=args.transport_tos, - nvme_error_stat=args.nvme_error_stat) + nvme_error_stat=args.nvme_error_stat, + rdma_srq_size=args.rdma_srq_size) p = subparsers.add_parser('bdev_nvme_set_options', help='Set options for the bdev nvme type. This is startup command.') @@ -634,6 +635,8 @@ if __name__ == "__main__": help="""IPv4 Type of Service value. Only applicable for RDMA transports. The default is 0 which means no TOS is applied.""", type=int) p.add_argument('-m', '--nvme-error-stat', help="Enable collecting NVMe error counts.", action='store_true') + p.add_argument('-q', '--rdma-srq-size', + help='Set the size of a shared rdma receive queue. Default: 0 (disabled)', type=int) p.set_defaults(func=bdev_nvme_set_options) diff --git a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c index 2fb15a1aa..162c147eb 100644 --- a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c +++ b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c @@ -52,6 +52,12 @@ DEFINE_RETURN_MOCK(spdk_nvme_ctrlr_get_memory_domains, int); DEFINE_STUB_V(spdk_jsonrpc_send_error_response, (struct spdk_jsonrpc_request *request, int error_code, const char *msg)); +DEFINE_STUB_V(spdk_nvme_transport_get_opts, (struct spdk_nvme_transport_opts *opts, + size_t opts_size)); + +DEFINE_STUB(spdk_nvme_transport_set_opts, int, (const struct spdk_nvme_transport_opts *opts, + size_t opts_size), 0); + int spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr, struct spdk_memory_domain **domains, int array_size)