diff --git a/CHANGELOG.md b/CHANGELOG.md index 00330fc02..023d2705a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,11 @@ a thinly provisioned logical volume. See documentation for complete details. ### nvme +Added an optional parameter `--io-queue-requests` to RPC `set_bdev_nvme_options`, which +can be used to change the number of requests allocated for one NVMe I/O queue. For +very big I/O size, e.g. 128MiB, with this option user will not get an error due to +limited requests in NVMe driver layer. + Added spdk_nvme_ctrlr_get_transport_id() to get the transport ID from a previously attached controller. diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 33dd83cf5..ca97a7394 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -1402,6 +1402,7 @@ timeout_us | Optional | number | Timeout for each command, retry_count | Optional | number | The number of attempts per I/O before an I/O fails nvme_adminq_poll_period_us | Optional | number | How often the admin queue is polled for asynchronous events in microseconds nvme_ioq_poll_period_us | Optional | number | How often I/O queues are polled for completions, in microseconds. Default: 0 (as fast as possible). +io_queue_requests | Optional | number | The number of requests allocated for each NVMe I/O queue. Default: 512. ### Example @@ -1414,7 +1415,8 @@ request: "retry_count": 5, "nvme_adminq_poll_period_us": 2000, "timeout_us": 10000000, - "action_on_timeout": "reset" + "action_on_timeout": "reset", + "io_queue_requests" : 2048, }, "jsonrpc": "2.0", "method": "set_bdev_nvme_options", diff --git a/lib/bdev/nvme/bdev_nvme.c b/lib/bdev/nvme/bdev_nvme.c index 0a15560d5..574ebd3cc 100644 --- a/lib/bdev/nvme/bdev_nvme.c +++ b/lib/bdev/nvme/bdev_nvme.c @@ -107,6 +107,7 @@ static struct spdk_bdev_nvme_opts g_opts = { .retry_count = SPDK_NVME_DEFAULT_RETRY_COUNT, .nvme_adminq_poll_period_us = 1000000ULL, .nvme_ioq_poll_period_us = 0, + .io_queue_requests = 0, }; #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL @@ -537,6 +538,8 @@ bdev_nvme_create_cb(void *io_device, void *ctx_buf) spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); opts.delay_pcie_doorbell = true; + opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); + g_opts.io_queue_requests = opts.io_queue_requests; ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, &opts, sizeof(opts)); @@ -1098,6 +1101,7 @@ spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) return 0; } + struct set_nvme_hotplug_ctx { uint64_t period_us; bool enabled; @@ -2118,6 +2122,7 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w) spdk_json_write_named_uint32(w, "retry_count", g_opts.retry_count); spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); + spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); spdk_json_write_object_end(w); spdk_json_write_object_end(w); diff --git a/lib/bdev/nvme/bdev_nvme.h b/lib/bdev/nvme/bdev_nvme.h index 150351dfc..fb44e2d8f 100644 --- a/lib/bdev/nvme/bdev_nvme.h +++ b/lib/bdev/nvme/bdev_nvme.h @@ -52,6 +52,7 @@ struct spdk_bdev_nvme_opts { uint32_t retry_count; uint64_t nvme_adminq_poll_period_us; uint64_t nvme_ioq_poll_period_us; + uint32_t io_queue_requests; }; typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, int rc); diff --git a/lib/bdev/nvme/bdev_nvme_rpc.c b/lib/bdev/nvme/bdev_nvme_rpc.c index fdbbd2219..7b8e3c93c 100644 --- a/lib/bdev/nvme/bdev_nvme_rpc.c +++ b/lib/bdev/nvme/bdev_nvme_rpc.c @@ -75,6 +75,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_options_decoders[] = {"retry_count", offsetof(struct spdk_bdev_nvme_opts, retry_count), spdk_json_decode_uint32, true}, {"nvme_adminq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_adminq_poll_period_us), spdk_json_decode_uint64, true}, {"nvme_ioq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_ioq_poll_period_us), spdk_json_decode_uint64, true}, + {"io_queue_requests", offsetof(struct spdk_bdev_nvme_opts, io_queue_requests), spdk_json_decode_uint32, true}, }; static void diff --git a/scripts/rpc.py b/scripts/rpc.py index 472573981..ee1b78952 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -298,7 +298,8 @@ if __name__ == "__main__": timeout_us=args.timeout_us, retry_count=args.retry_count, nvme_adminq_poll_period_us=args.nvme_adminq_poll_period_us, - nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us) + nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us, + io_queue_requests=args.io_queue_requests) p = subparsers.add_parser('set_bdev_nvme_options', help='Set options for the bdev nvme type. This is startup command.') @@ -312,6 +313,8 @@ if __name__ == "__main__": help='How often the admin queue is polled for asynchronous events', type=int) p.add_argument('-i', '--nvme-ioq-poll-period-us', help='How often to poll I/O queues for completions', type=int) + p.add_argument('-s', '--io-queue-requests', + help='The number of requests allocated for each NVMe I/O queue. Default: 512', type=int) p.set_defaults(func=set_bdev_nvme_options) def set_bdev_nvme_hotplug(args): diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 2a2c2bc15..5616a90a5 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -276,7 +276,7 @@ def delete_aio_bdev(client, name): def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry_count=None, - nvme_adminq_poll_period_us=None, nvme_ioq_poll_period_us=None): + nvme_adminq_poll_period_us=None, nvme_ioq_poll_period_us=None, io_queue_requests=None): """Set options for the bdev nvme. This is startup command. Args: @@ -285,6 +285,7 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry retry_count: The number of attempts per I/O when an I/O fails (optional) nvme_adminq_poll_period_us: How often the admin queue is polled for asynchronous events in microseconds (optional) nvme_ioq_poll_period_us: How often to poll I/O queues for completions in microseconds (optional) + io_queue_requests: The number of requests allocated for each NVMe I/O queue. Default: 512 (optional) """ params = {} @@ -303,6 +304,9 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry if nvme_ioq_poll_period_us: params['nvme_ioq_poll_period_us'] = nvme_ioq_poll_period_us + if io_queue_requests: + params['io_queue_requests'] = io_queue_requests + return client.call('set_bdev_nvme_options', params)