From 90520634bdb2b66639fe3629d27fe4fdc7c0bbf4 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Wed, 10 Jul 2019 01:13:31 -0400 Subject: [PATCH] bdev/nvme: configure the number of requests allocated for one NVMe I/O queue via RPC A single I/O may allocate more than one request, since splitting may be necessary to conform to the device's maximum transfer size, PRP list compatibility requirements, or driver-assisted striping. Very big I/O request sent from application may get error due to limited resources in NVMe driver layer, so here we add an optional parameter to make the parameter can be configured by users. Fix issue #745. Change-Id: I7824232c54865b052dcd0ec6e91484c3837fc2c4 Signed-off-by: Changpeng Liu Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/461182 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto --- CHANGELOG.md | 5 +++++ doc/jsonrpc.md | 4 +++- lib/bdev/nvme/bdev_nvme.c | 5 +++++ lib/bdev/nvme/bdev_nvme.h | 1 + lib/bdev/nvme/bdev_nvme_rpc.c | 1 + scripts/rpc.py | 5 ++++- scripts/rpc/bdev.py | 6 +++++- 7 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00330fc02..023d2705a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,11 @@ a thinly provisioned logical volume. See documentation for complete details. ### nvme +Added an optional parameter `--io-queue-requests` to RPC `set_bdev_nvme_options`, which +can be used to change the number of requests allocated for one NVMe I/O queue. For +very big I/O size, e.g. 128MiB, with this option user will not get an error due to +limited requests in NVMe driver layer. + Added spdk_nvme_ctrlr_get_transport_id() to get the transport ID from a previously attached controller. diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 33dd83cf5..ca97a7394 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -1402,6 +1402,7 @@ timeout_us | Optional | number | Timeout for each command, retry_count | Optional | number | The number of attempts per I/O before an I/O fails nvme_adminq_poll_period_us | Optional | number | How often the admin queue is polled for asynchronous events in microseconds nvme_ioq_poll_period_us | Optional | number | How often I/O queues are polled for completions, in microseconds. Default: 0 (as fast as possible). +io_queue_requests | Optional | number | The number of requests allocated for each NVMe I/O queue. Default: 512. ### Example @@ -1414,7 +1415,8 @@ request: "retry_count": 5, "nvme_adminq_poll_period_us": 2000, "timeout_us": 10000000, - "action_on_timeout": "reset" + "action_on_timeout": "reset", + "io_queue_requests" : 2048, }, "jsonrpc": "2.0", "method": "set_bdev_nvme_options", diff --git a/lib/bdev/nvme/bdev_nvme.c b/lib/bdev/nvme/bdev_nvme.c index 0a15560d5..574ebd3cc 100644 --- a/lib/bdev/nvme/bdev_nvme.c +++ b/lib/bdev/nvme/bdev_nvme.c @@ -107,6 +107,7 @@ static struct spdk_bdev_nvme_opts g_opts = { .retry_count = SPDK_NVME_DEFAULT_RETRY_COUNT, .nvme_adminq_poll_period_us = 1000000ULL, .nvme_ioq_poll_period_us = 0, + .io_queue_requests = 0, }; #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL @@ -537,6 +538,8 @@ bdev_nvme_create_cb(void *io_device, void *ctx_buf) spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); opts.delay_pcie_doorbell = true; + opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); + g_opts.io_queue_requests = opts.io_queue_requests; ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, &opts, sizeof(opts)); @@ -1098,6 +1101,7 @@ spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) return 0; } + struct set_nvme_hotplug_ctx { uint64_t period_us; bool enabled; @@ -2118,6 +2122,7 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w) spdk_json_write_named_uint32(w, "retry_count", g_opts.retry_count); spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); + spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); spdk_json_write_object_end(w); spdk_json_write_object_end(w); diff --git a/lib/bdev/nvme/bdev_nvme.h b/lib/bdev/nvme/bdev_nvme.h index 150351dfc..fb44e2d8f 100644 --- a/lib/bdev/nvme/bdev_nvme.h +++ b/lib/bdev/nvme/bdev_nvme.h @@ -52,6 +52,7 @@ struct spdk_bdev_nvme_opts { uint32_t retry_count; uint64_t nvme_adminq_poll_period_us; uint64_t nvme_ioq_poll_period_us; + uint32_t io_queue_requests; }; typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, int rc); diff --git a/lib/bdev/nvme/bdev_nvme_rpc.c b/lib/bdev/nvme/bdev_nvme_rpc.c index fdbbd2219..7b8e3c93c 100644 --- a/lib/bdev/nvme/bdev_nvme_rpc.c +++ b/lib/bdev/nvme/bdev_nvme_rpc.c @@ -75,6 +75,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_options_decoders[] = {"retry_count", offsetof(struct spdk_bdev_nvme_opts, retry_count), spdk_json_decode_uint32, true}, {"nvme_adminq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_adminq_poll_period_us), spdk_json_decode_uint64, true}, {"nvme_ioq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_ioq_poll_period_us), spdk_json_decode_uint64, true}, + {"io_queue_requests", offsetof(struct spdk_bdev_nvme_opts, io_queue_requests), spdk_json_decode_uint32, true}, }; static void diff --git a/scripts/rpc.py b/scripts/rpc.py index 472573981..ee1b78952 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -298,7 +298,8 @@ if __name__ == "__main__": timeout_us=args.timeout_us, retry_count=args.retry_count, nvme_adminq_poll_period_us=args.nvme_adminq_poll_period_us, - nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us) + nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us, + io_queue_requests=args.io_queue_requests) p = subparsers.add_parser('set_bdev_nvme_options', help='Set options for the bdev nvme type. This is startup command.') @@ -312,6 +313,8 @@ if __name__ == "__main__": help='How often the admin queue is polled for asynchronous events', type=int) p.add_argument('-i', '--nvme-ioq-poll-period-us', help='How often to poll I/O queues for completions', type=int) + p.add_argument('-s', '--io-queue-requests', + help='The number of requests allocated for each NVMe I/O queue. Default: 512', type=int) p.set_defaults(func=set_bdev_nvme_options) def set_bdev_nvme_hotplug(args): diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 2a2c2bc15..5616a90a5 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -276,7 +276,7 @@ def delete_aio_bdev(client, name): def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry_count=None, - nvme_adminq_poll_period_us=None, nvme_ioq_poll_period_us=None): + nvme_adminq_poll_period_us=None, nvme_ioq_poll_period_us=None, io_queue_requests=None): """Set options for the bdev nvme. This is startup command. Args: @@ -285,6 +285,7 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry retry_count: The number of attempts per I/O when an I/O fails (optional) nvme_adminq_poll_period_us: How often the admin queue is polled for asynchronous events in microseconds (optional) nvme_ioq_poll_period_us: How often to poll I/O queues for completions in microseconds (optional) + io_queue_requests: The number of requests allocated for each NVMe I/O queue. Default: 512 (optional) """ params = {} @@ -303,6 +304,9 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry if nvme_ioq_poll_period_us: params['nvme_ioq_poll_period_us'] = nvme_ioq_poll_period_us + if io_queue_requests: + params['io_queue_requests'] = io_queue_requests + return client.call('set_bdev_nvme_options', params)