QoS/Bdev: add the RPC support for the bandwidth rate limit

This patch added the support of RPC method to enable, adjust, disable the bandwidth rate limit on the bdev. And it can work together with the existing IOPS rate limit. The RPC method has been consolidated to support both IOPS and bandwidth rate limits as below: usage: rpc.py set_bdev_qos_limit [-h] [--rw_ios_per_sec RW_IOS_PER_SEC] [--rw_mbytes_per_sec RW_MBYTES_PER_SEC] name positional arguments: name Blockdev name to set QoS. Example: Malloc0 optional arguments: -h, --help show this help message and exit --rw_ios_per_sec RW_IOS_PER_SEC R/W IOs per second limit (>=10000, example: 20000). 0 means unlimited. --rw_mbytes_per_sec RW_MBYTES_PER_SEC R/W megabytes per second limit (>=10, example: 100). 0 means unlimited. Change-Id: I9c03cd635280add01801a81c6a6c02f0cf85bee1 Signed-off-by: GangCao <gang.cao@intel.com> Reviewed-on: https://review.gerrithub.io/416511 Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com> Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Ziye Yang <optimistyzy@gmail.com> Reviewed-by: Changpeng Liu <changpeng.liu@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Reviewed-by: Ben Walker <benjamin.walker@intel.com>
2018-06-21 22:15:02 -04:00 · 2018-06-21 22:15:02 -04:00 · 868c28cd13
commit 868c28cd13
parent d9ecb5724e
8 changed files with 124 additions and 48 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -35,6 +35,12 @@ currently marked as experimental.  Do not use in production.
 The RAID virtual bdev module is now always enabled by default.  The configure --with-raid and
 --without-raid options are now ignored and deprecated and will be removed in the next release.

+Enforcement of bandwidth limits for quality of service (QoS) has been added to the bdev layer.
+See the new [set_bdev_qos_limit](http://www.spdk.io/doc/jsonrpc.html#rpc_set_bdev_qos_limit)
+documentation for more details. The previous set_bdev_qos_limit_iops RPC method introduced at
+18.04 release has been deprecated. The new set_bdev_qos_limit RPC method can support both
+bandwidth and IOPS limits.
+
 ### Environment Abstraction Layer and Event Framework

 The size parameter of spdk_mem_map_translate is now a pointer. This allows the
--- a/doc/jsonrpc.md
+++ b/doc/jsonrpc.md
@ -230,6 +230,7 @@ Example response:
    "set_iscsi_options",
    "set_bdev_options",
    "set_bdev_qos_limit_iops",
+    "set_bdev_qos_limit",
    "delete_bdev",
    "get_bdevs",
    "get_bdevs_iostat",
@ -627,16 +628,17 @@ Example response:
 }
 ~~~

-## set_bdev_qos_limit_iops {#rpc_set_bdev_qos_limit_iops}
+## set_bdev_qos_limit {#rpc_set_bdev_qos_limit}

-Set an IOPS-based quality of service rate limit on a bdev.
+Set the quality of service rate limit on a bdev.

 ### Parameters

 Name                    | Optional | Type        | Description
 ----------------------- | -------- | ----------- | -----------
 name                    | Required | string      | Block device name
-ios_per_sec             | Required | number      | Number of I/Os per second to allow. 0 means unlimited.
+rw_ios_per_sec          | Optional | number      | Number of R/W I/Os per second to allow. 0 means unlimited.
+rw_mbytes_per_sec       | Optional | number      | Number of R/W megabytes per second to allow. 0 means unlimited.

 ### Example

@ -645,10 +647,11 @@ Example request:
 {
  "jsonrpc": "2.0",
  "id": 1,
-  "method": "set_bdev_qos_limit_iops",
+  "method": "set_bdev_qos_limit",
  "params": {
    "name": "Malloc0"
-    "ios_per_sec": 20000
+    "rw_ios_per_sec": 20000
+    "rw_mbytes_per_sec": 100
  }
 }
 ~~~
--- a/lib/bdev/bdev.c
+++ b/lib/bdev/bdev.c
@ -80,7 +80,7 @@ int __itt_init_ittlib(const char *, __itt_group_id);
 #define SPDK_BDEV_QOS_LIMIT_NOT_DEFINED		UINT64_MAX

 static const char *qos_conf_type[] = {"Limit_IOPS", "Limit_BPS"};
-static const char *qos_rpc_type[] = {"qos_ios_per_sec"};
+static const char *qos_rpc_type[] = {"rw_ios_per_sec", "rw_mbytes_per_sec"};

 TAILQ_HEAD(spdk_bdev_list, spdk_bdev);

@ -517,6 +517,35 @@ spdk_bdev_config_text(FILE *fp)
 	}
 }

+static void
+spdk_bdev_qos_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
+{
+	int i;
+	struct spdk_bdev_qos *qos = bdev->internal.qos;
+	uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];
+
+	if (!qos) {
+		return;
+	}
+
+	spdk_bdev_get_qos_rate_limits(bdev, limits);
+
+	spdk_json_write_object_begin(w);
+	spdk_json_write_named_string(w, "method", "set_bdev_qos_limit");
+	spdk_json_write_name(w, "params");
+
+	spdk_json_write_object_begin(w);
+	spdk_json_write_named_string(w, "name", bdev->name);
+	for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
+		if (limits[i] > 0) {
+			spdk_json_write_named_uint64(w, qos_rpc_type[i], limits[i]);
+		}
+	}
+	spdk_json_write_object_end(w);
+
+	spdk_json_write_object_end(w);
+}
+
 void
 spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w)
 {
@ -543,6 +572,8 @@ spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w)
 	}

 	TAILQ_FOREACH(bdev, &g_bdev_mgr.bdevs, internal.link) {
+		spdk_bdev_qos_config_json(bdev, w);
+
 		if (bdev->fn_table->write_config_json) {
 			bdev->fn_table->write_config_json(bdev, w);
 		}
@ -1953,6 +1984,10 @@ spdk_bdev_get_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits)
 			if (bdev->internal.qos->rate_limits[i].limit !=
 			    SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
 				limits[i] = bdev->internal.qos->rate_limits[i].limit;
+				if (_spdk_bdev_qos_is_iops_rate_limit(i) == false) {
+					/* Change from Byte to Megabyte which is user visible. */
+					limits[i] = limits[i] / 1024 / 1024;
+				}
 			}
 		}
 	}
--- a/lib/bdev/rpc/bdev_rpc.c
+++ b/lib/bdev/rpc/bdev_rpc.c
@ -220,6 +220,7 @@ spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w,
 {
 	struct spdk_bdev_alias *tmp;
 	uint64_t qos_limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];
+	int i;

 	spdk_json_write_object_begin(w);

@ -251,9 +252,14 @@ spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w,
 		spdk_json_write_named_string(w, "uuid", uuid_str);
 	}

+	spdk_json_write_name(w, "assigned_rate_limits");
+	spdk_json_write_object_begin(w);
 	spdk_bdev_get_qos_rate_limits(bdev, qos_limits);
-	spdk_json_write_name(w, spdk_bdev_get_qos_rpc_type(SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT));
-	spdk_json_write_uint64(w, qos_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT]);
+	for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
+		spdk_json_write_name(w, spdk_bdev_get_qos_rpc_type(i));
+		spdk_json_write_uint64(w, qos_limits[i]);
+	}
+	spdk_json_write_object_end(w);

 	spdk_json_write_name(w, "claimed");
 	spdk_json_write_bool(w, (bdev->internal.claim_module != NULL));
@ -484,31 +490,40 @@ SPDK_RPC_REGISTER("set_bdev_qd_sampling_period",
 		  spdk_rpc_set_bdev_qd_sampling_period,
 		  SPDK_RPC_RUNTIME)

-struct rpc_set_bdev_qos_limit_iops {
+struct rpc_set_bdev_qos_limit {
 	char		*name;
-	uint64_t	ios_per_sec;
+	uint64_t	limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];
 };

 static void
-free_rpc_set_bdev_qos_limit_iops(struct rpc_set_bdev_qos_limit_iops *r)
+free_rpc_set_bdev_qos_limit(struct rpc_set_bdev_qos_limit *r)
 {
 	free(r->name);
 }

-static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_iops_decoders[] = {
-	{"name", offsetof(struct rpc_set_bdev_qos_limit_iops, name), spdk_json_decode_string},
-	{"ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit_iops, ios_per_sec), spdk_json_decode_uint64},
+static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_decoders[] = {
+	{"name", offsetof(struct rpc_set_bdev_qos_limit, name), spdk_json_decode_string},
+	{
+		"rw_ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit,
+					   limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT]),
+		spdk_json_decode_uint64, true
+	},
+	{
+		"rw_mbytes_per_sec", offsetof(struct rpc_set_bdev_qos_limit,
+					      limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT]),
+		spdk_json_decode_uint64, true
+	},
 };

 static void
-spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status)
+spdk_rpc_set_bdev_qos_limit_complete(void *cb_arg, int status)
 {
 	struct spdk_jsonrpc_request *request = cb_arg;
 	struct spdk_json_write_ctx *w;

 	if (status != 0) {
 		spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
-						     "Failed to configure IOPS limit: %s",
+						     "Failed to configure rate limit: %s",
 						     spdk_strerror(-status));
 		return;
 	}
@ -523,15 +538,16 @@ spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status)
 }

 static void
-spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request,
-				 const struct spdk_json_val *params)
+spdk_rpc_set_bdev_qos_limit(struct spdk_jsonrpc_request *request,
+			    const struct spdk_json_val *params)
 {
-	struct rpc_set_bdev_qos_limit_iops req = {NULL, UINT64_MAX};
+	struct rpc_set_bdev_qos_limit req = {NULL, {UINT64_MAX, UINT64_MAX}};
 	struct spdk_bdev *bdev;
-	uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {UINT64_MAX, UINT64_MAX};
+	bool valid_limit = false;
+	int i;

-	if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_iops_decoders,
-				    SPDK_COUNTOF(rpc_set_bdev_qos_limit_iops_decoders),
+	if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_decoders,
+				    SPDK_COUNTOF(rpc_set_bdev_qos_limit_decoders),
 				    &req)) {
 		SPDK_ERRLOG("spdk_json_decode_object failed\n");
 		goto invalid;
@ -545,23 +561,27 @@ spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request,
 		goto exit;
 	}

-	if (req.ios_per_sec == UINT64_MAX) {
-		SPDK_ERRLOG("invalid rate limits set\n");
+	for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
+		if (req.limits[i] != UINT64_MAX) {
+			valid_limit = true;
+		}
+	}
+
+	if (valid_limit == false) {
+		SPDK_ERRLOG("no rate limits specified\n");
 		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
-						 "Invalid rate limits");
+						 "No rate limits specified");
 		goto exit;
 	}

-	free_rpc_set_bdev_qos_limit_iops(&req);
-	limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = req.ios_per_sec;
-	spdk_bdev_set_qos_rate_limits(bdev, limits, spdk_rpc_set_bdev_qos_limit_iops_complete,
-				      request);
+	free_rpc_set_bdev_qos_limit(&req);
+	spdk_bdev_set_qos_rate_limits(bdev, req.limits, spdk_rpc_set_bdev_qos_limit_complete, request);
 	return;

 invalid:
 	spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");
 exit:
-	free_rpc_set_bdev_qos_limit_iops(&req);
+	free_rpc_set_bdev_qos_limit(&req);
 }

-SPDK_RPC_REGISTER("set_bdev_qos_limit_iops", spdk_rpc_set_bdev_qos_limit_iops, SPDK_RPC_RUNTIME)
+SPDK_RPC_REGISTER("set_bdev_qos_limit", spdk_rpc_set_bdev_qos_limit, SPDK_RPC_RUNTIME)
--- a/scripts/rpc.py
+++ b/scripts/rpc.py
@ -449,16 +449,21 @@ if __name__ == "__main__":
    p.set_defaults(func=set_bdev_qd_sampling_period)

    @call_cmd
-    def set_bdev_qos_limit_iops(args):
-        rpc.bdev.set_bdev_qos_limit_iops(args.client,
-                                         name=args.name,
-                                         ios_per_sec=args.ios_per_sec)
+    def set_bdev_qos_limit(args):
+        rpc.bdev.set_bdev_qos_limit(args.client,
+                                    name=args.name,
+                                    rw_ios_per_sec=args.rw_ios_per_sec,
+                                    rw_mbytes_per_sec=args.rw_mbytes_per_sec)

-    p = subparsers.add_parser('set_bdev_qos_limit_iops', help='Set QoS IOPS limit on a blockdev')
+    p = subparsers.add_parser('set_bdev_qos_limit', help='Set QoS rate limit on a blockdev')
    p.add_argument('name', help='Blockdev name to set QoS. Example: Malloc0')
-    p.add_argument('ios_per_sec',
-                   help='IOs per second limit (>=10000, example: 20000). 0 means unlimited.', type=int)
-    p.set_defaults(func=set_bdev_qos_limit_iops)
+    p.add_argument('--rw_ios_per_sec',
+                   help='R/W IOs per second limit (>=10000, example: 20000). 0 means unlimited.',
+                   type=int, required=False)
+    p.add_argument('--rw_mbytes_per_sec',
+                   help="R/W megabytes per second limit (>=10, example: 100). 0 means unlimited.",
+                   type=int, required=False)
+    p.set_defaults(func=set_bdev_qos_limit)

    @call_cmd
    def bdev_inject_error(args):
--- a/scripts/rpc/bdev.py
+++ b/scripts/rpc/bdev.py
@ -500,17 +500,21 @@ def set_bdev_qd_sampling_period(client, name, period):
    return client.call('set_bdev_qd_sampling_period', params)


-def set_bdev_qos_limit_iops(client, name, ios_per_sec):
-    """Set QoS IOPS limit on a block device.
+def set_bdev_qos_limit(client, name, rw_ios_per_sec=None, rw_mbytes_per_sec=None):
+    """Set QoS rate limit on a block device.

    Args:
        name: name of block device
-        ios_per_sec: IOs per second limit (>=10000, example: 20000). 0 means unlimited.
+        rw_ios_per_sec: R/W IOs per second limit (>=10000, example: 20000). 0 means unlimited.
+        rw_mbytes_per_sec: R/W megabytes per second limit (>=10, example: 100). 0 means unlimited.
    """
    params = {}
    params['name'] = name
-    params['ios_per_sec'] = ios_per_sec
-    return client.call('set_bdev_qos_limit_iops', params)
+    if rw_ios_per_sec is not None:
+        params['rw_ios_per_sec'] = rw_ios_per_sec
+    if rw_mbytes_per_sec is not None:
+        params['rw_mbytes_per_sec'] = rw_mbytes_per_sec
+    return client.call('set_bdev_qos_limit', params)


 def apply_firmware(client, bdev_name, filename):
--- a/test/iscsi_tgt/qos/qos.sh
+++ b/test/iscsi_tgt/qos/qos.sh
@ -76,15 +76,15 @@ iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT
 trap "iscsicleanup; killprocess $pid; exit 1" SIGINT SIGTERM EXIT

 # Limit the I/O rate by RPC, then confirm the observed rate matches.
-$rpc_py set_bdev_qos_limit_iops Malloc0 $IOPS_LIMIT
+$rpc_py set_bdev_qos_limit Malloc0 --rw_ios_per_sec $IOPS_LIMIT
 check_qos_works_well true $IOPS_LIMIT Malloc0

 # Now disable the rate limiting, and confirm the observed rate is not limited anymore.
-$rpc_py set_bdev_qos_limit_iops Malloc0 0
+$rpc_py set_bdev_qos_limit Malloc0 --rw_ios_per_sec 0
 check_qos_works_well false $IOPS_LIMIT Malloc0

 # Limit the I/O rate again.
-$rpc_py set_bdev_qos_limit_iops Malloc0 $IOPS_LIMIT
+$rpc_py set_bdev_qos_limit Malloc0 --rw_ios_per_sec $IOPS_LIMIT
 check_qos_works_well true $IOPS_LIMIT Malloc0
 echo "I/O rate limiting tests successful"

--- a/test/spdkcli/match_files/spdkcli_details_vhost.test.match
+++ b/test/spdkcli/match_files/spdkcli_details_vhost.test.match
@ -1,5 +1,9 @@
 {
  "aliases": [],
+  "assigned_rate_limits": {
+    "rw_ios_per_sec": $(N),
+    "rw_mbytes_per_sec": $(N)
+  },
  "block_size": $(N),
  "claimed": false,
  "driver_specific": {
@ -11,7 +15,6 @@
  "name": "Nvme0n1p0",
  "num_blocks": $(N),
  "product_name": "Split Disk",
-  "qos_ios_per_sec": 0,
  "supported_io_types": {
    "flush": $(S),
    "nvme_admin": $(S),