diff --git a/CHANGELOG.md b/CHANGELOG.md index b510642b3..724066136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,12 @@ currently marked as experimental. Do not use in production. The RAID virtual bdev module is now always enabled by default. The configure --with-raid and --without-raid options are now ignored and deprecated and will be removed in the next release. +Enforcement of bandwidth limits for quality of service (QoS) has been added to the bdev layer. +See the new [set_bdev_qos_limit](http://www.spdk.io/doc/jsonrpc.html#rpc_set_bdev_qos_limit) +documentation for more details. The previous set_bdev_qos_limit_iops RPC method introduced at +18.04 release has been deprecated. The new set_bdev_qos_limit RPC method can support both +bandwidth and IOPS limits. + ### Environment Abstraction Layer and Event Framework The size parameter of spdk_mem_map_translate is now a pointer. This allows the diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 3045e087e..566d8974b 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -230,6 +230,7 @@ Example response: "set_iscsi_options", "set_bdev_options", "set_bdev_qos_limit_iops", + "set_bdev_qos_limit", "delete_bdev", "get_bdevs", "get_bdevs_iostat", @@ -627,16 +628,17 @@ Example response: } ~~~ -## set_bdev_qos_limit_iops {#rpc_set_bdev_qos_limit_iops} +## set_bdev_qos_limit {#rpc_set_bdev_qos_limit} -Set an IOPS-based quality of service rate limit on a bdev. +Set the quality of service rate limit on a bdev. ### Parameters Name | Optional | Type | Description ----------------------- | -------- | ----------- | ----------- name | Required | string | Block device name -ios_per_sec | Required | number | Number of I/Os per second to allow. 0 means unlimited. +rw_ios_per_sec | Optional | number | Number of R/W I/Os per second to allow. 0 means unlimited. +rw_mbytes_per_sec | Optional | number | Number of R/W megabytes per second to allow. 0 means unlimited. ### Example @@ -645,10 +647,11 @@ Example request: { "jsonrpc": "2.0", "id": 1, - "method": "set_bdev_qos_limit_iops", + "method": "set_bdev_qos_limit", "params": { "name": "Malloc0" - "ios_per_sec": 20000 + "rw_ios_per_sec": 20000 + "rw_mbytes_per_sec": 100 } } ~~~ diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index d5d82f87b..3a4fef817 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -80,7 +80,7 @@ int __itt_init_ittlib(const char *, __itt_group_id); #define SPDK_BDEV_QOS_LIMIT_NOT_DEFINED UINT64_MAX static const char *qos_conf_type[] = {"Limit_IOPS", "Limit_BPS"}; -static const char *qos_rpc_type[] = {"qos_ios_per_sec"}; +static const char *qos_rpc_type[] = {"rw_ios_per_sec", "rw_mbytes_per_sec"}; TAILQ_HEAD(spdk_bdev_list, spdk_bdev); @@ -517,6 +517,35 @@ spdk_bdev_config_text(FILE *fp) } } +static void +spdk_bdev_qos_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +{ + int i; + struct spdk_bdev_qos *qos = bdev->internal.qos; + uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES]; + + if (!qos) { + return; + } + + spdk_bdev_get_qos_rate_limits(bdev, limits); + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "set_bdev_qos_limit"); + spdk_json_write_name(w, "params"); + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "name", bdev->name); + for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { + if (limits[i] > 0) { + spdk_json_write_named_uint64(w, qos_rpc_type[i], limits[i]); + } + } + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); +} + void spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w) { @@ -543,6 +572,8 @@ spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w) } TAILQ_FOREACH(bdev, &g_bdev_mgr.bdevs, internal.link) { + spdk_bdev_qos_config_json(bdev, w); + if (bdev->fn_table->write_config_json) { bdev->fn_table->write_config_json(bdev, w); } @@ -1953,6 +1984,10 @@ spdk_bdev_get_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits) if (bdev->internal.qos->rate_limits[i].limit != SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) { limits[i] = bdev->internal.qos->rate_limits[i].limit; + if (_spdk_bdev_qos_is_iops_rate_limit(i) == false) { + /* Change from Byte to Megabyte which is user visible. */ + limits[i] = limits[i] / 1024 / 1024; + } } } } diff --git a/lib/bdev/rpc/bdev_rpc.c b/lib/bdev/rpc/bdev_rpc.c index 0e2156c56..1989f6d27 100644 --- a/lib/bdev/rpc/bdev_rpc.c +++ b/lib/bdev/rpc/bdev_rpc.c @@ -220,6 +220,7 @@ spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w, { struct spdk_bdev_alias *tmp; uint64_t qos_limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES]; + int i; spdk_json_write_object_begin(w); @@ -251,9 +252,14 @@ spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w, spdk_json_write_named_string(w, "uuid", uuid_str); } + spdk_json_write_name(w, "assigned_rate_limits"); + spdk_json_write_object_begin(w); spdk_bdev_get_qos_rate_limits(bdev, qos_limits); - spdk_json_write_name(w, spdk_bdev_get_qos_rpc_type(SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT)); - spdk_json_write_uint64(w, qos_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT]); + for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { + spdk_json_write_name(w, spdk_bdev_get_qos_rpc_type(i)); + spdk_json_write_uint64(w, qos_limits[i]); + } + spdk_json_write_object_end(w); spdk_json_write_name(w, "claimed"); spdk_json_write_bool(w, (bdev->internal.claim_module != NULL)); @@ -484,31 +490,40 @@ SPDK_RPC_REGISTER("set_bdev_qd_sampling_period", spdk_rpc_set_bdev_qd_sampling_period, SPDK_RPC_RUNTIME) -struct rpc_set_bdev_qos_limit_iops { +struct rpc_set_bdev_qos_limit { char *name; - uint64_t ios_per_sec; + uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES]; }; static void -free_rpc_set_bdev_qos_limit_iops(struct rpc_set_bdev_qos_limit_iops *r) +free_rpc_set_bdev_qos_limit(struct rpc_set_bdev_qos_limit *r) { free(r->name); } -static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_iops_decoders[] = { - {"name", offsetof(struct rpc_set_bdev_qos_limit_iops, name), spdk_json_decode_string}, - {"ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit_iops, ios_per_sec), spdk_json_decode_uint64}, +static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_decoders[] = { + {"name", offsetof(struct rpc_set_bdev_qos_limit, name), spdk_json_decode_string}, + { + "rw_ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit, + limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT]), + spdk_json_decode_uint64, true + }, + { + "rw_mbytes_per_sec", offsetof(struct rpc_set_bdev_qos_limit, + limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT]), + spdk_json_decode_uint64, true + }, }; static void -spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status) +spdk_rpc_set_bdev_qos_limit_complete(void *cb_arg, int status) { struct spdk_jsonrpc_request *request = cb_arg; struct spdk_json_write_ctx *w; if (status != 0) { spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Failed to configure IOPS limit: %s", + "Failed to configure rate limit: %s", spdk_strerror(-status)); return; } @@ -523,15 +538,16 @@ spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status) } static void -spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) +spdk_rpc_set_bdev_qos_limit(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) { - struct rpc_set_bdev_qos_limit_iops req = {NULL, UINT64_MAX}; + struct rpc_set_bdev_qos_limit req = {NULL, {UINT64_MAX, UINT64_MAX}}; struct spdk_bdev *bdev; - uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {UINT64_MAX, UINT64_MAX}; + bool valid_limit = false; + int i; - if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_iops_decoders, - SPDK_COUNTOF(rpc_set_bdev_qos_limit_iops_decoders), + if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_decoders, + SPDK_COUNTOF(rpc_set_bdev_qos_limit_decoders), &req)) { SPDK_ERRLOG("spdk_json_decode_object failed\n"); goto invalid; @@ -545,23 +561,27 @@ spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request, goto exit; } - if (req.ios_per_sec == UINT64_MAX) { - SPDK_ERRLOG("invalid rate limits set\n"); + for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { + if (req.limits[i] != UINT64_MAX) { + valid_limit = true; + } + } + + if (valid_limit == false) { + SPDK_ERRLOG("no rate limits specified\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid rate limits"); + "No rate limits specified"); goto exit; } - free_rpc_set_bdev_qos_limit_iops(&req); - limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = req.ios_per_sec; - spdk_bdev_set_qos_rate_limits(bdev, limits, spdk_rpc_set_bdev_qos_limit_iops_complete, - request); + free_rpc_set_bdev_qos_limit(&req); + spdk_bdev_set_qos_rate_limits(bdev, req.limits, spdk_rpc_set_bdev_qos_limit_complete, request); return; invalid: spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); exit: - free_rpc_set_bdev_qos_limit_iops(&req); + free_rpc_set_bdev_qos_limit(&req); } -SPDK_RPC_REGISTER("set_bdev_qos_limit_iops", spdk_rpc_set_bdev_qos_limit_iops, SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER("set_bdev_qos_limit", spdk_rpc_set_bdev_qos_limit, SPDK_RPC_RUNTIME) diff --git a/scripts/rpc.py b/scripts/rpc.py index 7db05239c..10772a0cc 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -449,16 +449,21 @@ if __name__ == "__main__": p.set_defaults(func=set_bdev_qd_sampling_period) @call_cmd - def set_bdev_qos_limit_iops(args): - rpc.bdev.set_bdev_qos_limit_iops(args.client, - name=args.name, - ios_per_sec=args.ios_per_sec) + def set_bdev_qos_limit(args): + rpc.bdev.set_bdev_qos_limit(args.client, + name=args.name, + rw_ios_per_sec=args.rw_ios_per_sec, + rw_mbytes_per_sec=args.rw_mbytes_per_sec) - p = subparsers.add_parser('set_bdev_qos_limit_iops', help='Set QoS IOPS limit on a blockdev') + p = subparsers.add_parser('set_bdev_qos_limit', help='Set QoS rate limit on a blockdev') p.add_argument('name', help='Blockdev name to set QoS. Example: Malloc0') - p.add_argument('ios_per_sec', - help='IOs per second limit (>=10000, example: 20000). 0 means unlimited.', type=int) - p.set_defaults(func=set_bdev_qos_limit_iops) + p.add_argument('--rw_ios_per_sec', + help='R/W IOs per second limit (>=10000, example: 20000). 0 means unlimited.', + type=int, required=False) + p.add_argument('--rw_mbytes_per_sec', + help="R/W megabytes per second limit (>=10, example: 100). 0 means unlimited.", + type=int, required=False) + p.set_defaults(func=set_bdev_qos_limit) @call_cmd def bdev_inject_error(args): diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 73891cb3f..6c7d0ecd3 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -500,17 +500,21 @@ def set_bdev_qd_sampling_period(client, name, period): return client.call('set_bdev_qd_sampling_period', params) -def set_bdev_qos_limit_iops(client, name, ios_per_sec): - """Set QoS IOPS limit on a block device. +def set_bdev_qos_limit(client, name, rw_ios_per_sec=None, rw_mbytes_per_sec=None): + """Set QoS rate limit on a block device. Args: name: name of block device - ios_per_sec: IOs per second limit (>=10000, example: 20000). 0 means unlimited. + rw_ios_per_sec: R/W IOs per second limit (>=10000, example: 20000). 0 means unlimited. + rw_mbytes_per_sec: R/W megabytes per second limit (>=10, example: 100). 0 means unlimited. """ params = {} params['name'] = name - params['ios_per_sec'] = ios_per_sec - return client.call('set_bdev_qos_limit_iops', params) + if rw_ios_per_sec is not None: + params['rw_ios_per_sec'] = rw_ios_per_sec + if rw_mbytes_per_sec is not None: + params['rw_mbytes_per_sec'] = rw_mbytes_per_sec + return client.call('set_bdev_qos_limit', params) def apply_firmware(client, bdev_name, filename): diff --git a/test/iscsi_tgt/qos/qos.sh b/test/iscsi_tgt/qos/qos.sh index a5cd36967..da12f8f85 100755 --- a/test/iscsi_tgt/qos/qos.sh +++ b/test/iscsi_tgt/qos/qos.sh @@ -76,15 +76,15 @@ iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT trap "iscsicleanup; killprocess $pid; exit 1" SIGINT SIGTERM EXIT # Limit the I/O rate by RPC, then confirm the observed rate matches. -$rpc_py set_bdev_qos_limit_iops Malloc0 $IOPS_LIMIT +$rpc_py set_bdev_qos_limit Malloc0 --rw_ios_per_sec $IOPS_LIMIT check_qos_works_well true $IOPS_LIMIT Malloc0 # Now disable the rate limiting, and confirm the observed rate is not limited anymore. -$rpc_py set_bdev_qos_limit_iops Malloc0 0 +$rpc_py set_bdev_qos_limit Malloc0 --rw_ios_per_sec 0 check_qos_works_well false $IOPS_LIMIT Malloc0 # Limit the I/O rate again. -$rpc_py set_bdev_qos_limit_iops Malloc0 $IOPS_LIMIT +$rpc_py set_bdev_qos_limit Malloc0 --rw_ios_per_sec $IOPS_LIMIT check_qos_works_well true $IOPS_LIMIT Malloc0 echo "I/O rate limiting tests successful" diff --git a/test/spdkcli/match_files/spdkcli_details_vhost.test.match b/test/spdkcli/match_files/spdkcli_details_vhost.test.match index 4189e106f..8cbd9e801 100644 --- a/test/spdkcli/match_files/spdkcli_details_vhost.test.match +++ b/test/spdkcli/match_files/spdkcli_details_vhost.test.match @@ -1,5 +1,9 @@ { "aliases": [], + "assigned_rate_limits": { + "rw_ios_per_sec": $(N), + "rw_mbytes_per_sec": $(N) + }, "block_size": $(N), "claimed": false, "driver_specific": { @@ -11,7 +15,6 @@ "name": "Nvme0n1p0", "num_blocks": $(N), "product_name": "Split Disk", - "qos_ios_per_sec": 0, "supported_io_types": { "flush": $(S), "nvme_admin": $(S),