From ffba4fdbc3273b68d048b1d364ac1883475a7cdb Mon Sep 17 00:00:00 2001 From: GangCao Date: Fri, 29 Dec 2017 03:02:08 -0500 Subject: [PATCH] bdev/qos: add RPC method to set QoS at runtime This patch adds a new RPC method to configure QoS on bdev at runtime. For example: set_bdev_qos_limit_iops Malloc0 20000 --> Enable QoS on this block device with 20000 IOPS rate limiting. Change-Id: I1ee8b313b769fb5a664820f4ba827e0230be4b5d Signed-off-by: GangCao Reviewed-on: https://review.gerrithub.io/393255 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Ben Walker --- doc/jsonrpc.md | 34 +++++++++++ include/spdk/bdev.h | 11 ++++ lib/bdev/bdev.c | 127 ++++++++++++++++++++++++++++++++++++---- lib/bdev/rpc/bdev_rpc.c | 71 ++++++++++++++++++++++ scripts/rpc.py | 9 +++ scripts/rpc/bdev.py | 7 +++ 6 files changed, 249 insertions(+), 10 deletions(-) diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index ee3eb0db1..b0dfa096f 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -172,6 +172,40 @@ Name | Optional | Type | Description ----------------------- | -------- | ----------- | ----------- name | Required | string | Block device name +## set_bdev_qos_limit_iops {#rpc_set_bdev_qos_limit_iops} + +Set an IOPS-based quality of service rate limit on a bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Block device name +ios_per_sec | Required | number | Number of I/Os per second to allow + +### Example + +Example request: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "set_bdev_qos_limit_iops", + "params": { + "name": "Malloc0" + "ios_per_sec": 20000 + } +} +~~~ + +Example response: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ # NVMe-oF Target {#jsonrpc_components_nvmf_tgt} diff --git a/include/spdk/bdev.h b/include/spdk/bdev.h index 60a2d716d..2d5d26503 100644 --- a/include/spdk/bdev.h +++ b/include/spdk/bdev.h @@ -659,6 +659,17 @@ int spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel * int spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, spdk_bdev_io_completion_cb cb, void *cb_arg); +/** + * Set an IOPS-based quality of service rate limit on a bdev. + * + * \param bdev Block device. + * \param ios_per_sec I/O per second limit. + * \param cb_fn Callback function to be called when the QoS limit has been updated. + * \param cb_arg Argument to pass to cb_fn. + */ +void spdk_bdev_set_qos_limit_iops(struct spdk_bdev *bdev, uint64_t ios_per_sec, + void (*cb_fn)(void *cb_arg, int status), void *cb_arg); + /** * Submit an NVMe Admin command to the bdev. This passes directly through * the block layer to the device. Support for NVMe passthru is optional, diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 5b501d784..342e7a258 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -1109,6 +1109,23 @@ spdk_bdev_qos_channel_create(struct spdk_bdev *bdev) return 0; } +/* Caller must hold bdev->mutex */ +static int +_spdk_bdev_enable_qos(struct spdk_bdev *bdev, struct spdk_bdev_channel *ch) +{ + /* Rate limiting on this bdev enabled */ + if (bdev->ios_per_sec) { + if (bdev->qos_channel == NULL) { + if (spdk_bdev_qos_channel_create(bdev) != 0) { + return -1; + } + } + ch->flags |= BDEV_CH_QOS_ENABLED; + } + + return 0; +} + static int spdk_bdev_channel_create(void *io_device, void *ctx_buf) { @@ -1138,16 +1155,10 @@ spdk_bdev_channel_create(void *io_device, void *ctx_buf) pthread_mutex_lock(&bdev->mutex); - /* Rate limiting on this bdev enabled */ - if (bdev->ios_per_sec) { - if (bdev->qos_channel == NULL) { - if (spdk_bdev_qos_channel_create(bdev) != 0) { - _spdk_bdev_channel_destroy_resource(ch); - pthread_mutex_unlock(&bdev->mutex); - return -1; - } - } - ch->flags |= BDEV_CH_QOS_ENABLED; + if (_spdk_bdev_enable_qos(bdev, ch)) { + _spdk_bdev_channel_destroy_resource(ch); + pthread_mutex_unlock(&bdev->mutex); + return -1; } bdev->channel_count++; @@ -2864,4 +2875,100 @@ spdk_bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io, bool success, void *c spdk_bdev_io_submit(bdev_io); } +struct set_qos_limit_ctx { + void (*cb_fn)(void *cb_arg, int status); + void *cb_arg; + struct spdk_bdev *bdev; +}; + +static void +_spdk_bdev_set_qos_limit_done(struct set_qos_limit_ctx *ctx, int status) +{ + ctx->cb_fn(ctx->cb_arg, status); + free(ctx); +} + +static void +_spdk_bdev_update_qos_limit_iops_msg(void *cb_arg) +{ + struct set_qos_limit_ctx *ctx = cb_arg; + struct spdk_bdev *bdev = ctx->bdev; + + /* + * There is possibility that the QoS channel has been destroyed + * when processing this message. Have a check here as the QoS + * channel is protected through the critical section. + */ + if (bdev->qos_channel) { + spdk_bdev_qos_update_max_ios_per_timeslice(bdev->qos_channel); + } + + _spdk_bdev_set_qos_limit_done(ctx, 0); +} + +static void +_spdk_bdev_enable_qos_msg(struct spdk_io_channel_iter *i) +{ + void *io_device = spdk_io_channel_iter_get_io_device(i); + struct spdk_bdev *bdev = __bdev_from_io_dev(io_device); + struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); + struct spdk_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(ch); + int rc; + + pthread_mutex_lock(&bdev->mutex); + rc = _spdk_bdev_enable_qos(bdev, bdev_ch); + pthread_mutex_unlock(&bdev->mutex); + + spdk_for_each_channel_continue(i, rc); +} + +static void +_spdk_bdev_enable_qos_done(struct spdk_io_channel_iter *i, int status) +{ + struct set_qos_limit_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + + _spdk_bdev_set_qos_limit_done(ctx, status); +} + +void +spdk_bdev_set_qos_limit_iops(struct spdk_bdev *bdev, uint64_t ios_per_sec, + void (*cb_fn)(void *cb_arg, int status), void *cb_arg) +{ + struct set_qos_limit_ctx *ctx; + + if (ios_per_sec == 0 || ios_per_sec % SPDK_BDEV_QOS_MIN_IOS_PER_SEC) { + SPDK_ERRLOG("Requested ios_per_sec limit %" PRIu64 " is not a multiple of %u\n", + ios_per_sec, SPDK_BDEV_QOS_MIN_IOS_PER_SEC); + cb_fn(cb_arg, -EINVAL); + return; + } + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + cb_fn(cb_arg, -ENOMEM); + return; + } + + ctx->cb_fn = cb_fn; + ctx->cb_arg = cb_arg; + + pthread_mutex_lock(&bdev->mutex); + bdev->ios_per_sec = ios_per_sec; + if (bdev->qos_thread) { + /* + * QoS is already enabled, so just update the limit information on the QoS thread. + */ + ctx->bdev = bdev; + spdk_thread_send_msg(bdev->qos_thread, _spdk_bdev_update_qos_limit_iops_msg, ctx); + pthread_mutex_unlock(&bdev->mutex); + return; + } + pthread_mutex_unlock(&bdev->mutex); + + /* Enable QoS on all channels. */ + spdk_for_each_channel(__bdev_to_io_dev(bdev), + _spdk_bdev_enable_qos_msg, ctx, + _spdk_bdev_enable_qos_done); +} + SPDK_LOG_REGISTER_COMPONENT("bdev", SPDK_LOG_BDEV) diff --git a/lib/bdev/rpc/bdev_rpc.c b/lib/bdev/rpc/bdev_rpc.c index f61972475..790160542 100644 --- a/lib/bdev/rpc/bdev_rpc.c +++ b/lib/bdev/rpc/bdev_rpc.c @@ -304,3 +304,74 @@ invalid: free_rpc_delete_bdev(&req); } SPDK_RPC_REGISTER("delete_bdev", spdk_rpc_delete_bdev) + + +struct rpc_set_bdev_qos_limit_iops { + char *name; + uint64_t ios_per_sec; +}; + +static void +free_rpc_set_bdev_qos_limit_iops(struct rpc_set_bdev_qos_limit_iops *r) +{ + free(r->name); +} + +static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_iops_decoders[] = { + {"name", offsetof(struct rpc_set_bdev_qos_limit_iops, name), spdk_json_decode_string}, + {"ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit_iops, ios_per_sec), spdk_json_decode_uint64}, +}; + +static void +spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status) +{ + struct spdk_jsonrpc_request *request = cb_arg; + struct spdk_json_write_ctx *w; + + if (status != 0) { + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Failed to configure IOPS limit: %s", + spdk_strerror(-status)); + return; + } + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} + +static void +spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_set_bdev_qos_limit_iops req = {}; + struct spdk_bdev *bdev; + + if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_iops_decoders, + SPDK_COUNTOF(rpc_set_bdev_qos_limit_iops_decoders), + &req)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + goto invalid; + } + + bdev = spdk_bdev_get_by_name(req.name); + if (bdev == NULL) { + SPDK_ERRLOG("bdev '%s' does not exist\n", req.name); + goto invalid; + } + + free_rpc_set_bdev_qos_limit_iops(&req); + spdk_bdev_set_qos_limit_iops(bdev, req.ios_per_sec, + spdk_rpc_set_bdev_qos_limit_iops_complete, request); + return; + +invalid: + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); + free_rpc_set_bdev_qos_limit_iops(&req); +} + +SPDK_RPC_REGISTER("set_bdev_qos_limit_iops", spdk_rpc_set_bdev_qos_limit_iops) diff --git a/scripts/rpc.py b/scripts/rpc.py index 5637bb132..c722949d9 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -198,6 +198,15 @@ if __name__ == "__main__": 'bdev_name', help='Blockdev name to be deleted. Example: Malloc0.') p.set_defaults(func=delete_bdev) + @call_cmd + def set_bdev_qos_limit_iops(args): + rpc.bdev.set_bdev_qos_limit_iops(args.client, args) + + p = subparsers.add_parser('set_bdev_qos_limit_iops', help='Set QoS IOPS limit on a blockdev') + p.add_argument('name', help='Blockdev name to set QoS. Example: Malloc0') + p.add_argument('ios_per_sec', help='IOs per second limit (>=10000). Example: 20000', type=int) + p.set_defaults(func=set_bdev_qos_limit_iops) + @call_cmd def bdev_inject_error(args): rpc.bdev.bdev_inject_error(args.client, args) diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index f2b975d3b..5fb84e068 100755 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -119,6 +119,13 @@ def bdev_inject_error(client, args): return client.call('bdev_inject_error', params) +def set_bdev_qos_limit_iops(client, args): + params = {} + params['name'] = args.name + params['ios_per_sec'] = args.ios_per_sec + return client.call('set_bdev_qos_limit_iops', params) + + def apply_firmware(client, args): params = { 'filename': args.filename,