bdev/delay: make latencies reconfigurable.

This will allow us to do interesting things in tests like configure a
bdev with a really long delay, and allow I/O to build up on a target to
see what happens, then decrease the latency to allow traffic to flow
through normally without ever having to delete and reconfigure the bdev.

Change-Id: Ibcb1101f8eed9fe3094ba239110cb4e49ace6554
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/464454
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

CHANGELOG.md

@@ -7,6 +7,11 @@
 Portals may no longer be associated with a cpumask. The scheduling of
 connections is moving to a more dynamic model.
 
+### delay bdev
+
+The `bdev_delay_update_latency` has been added to allow users to update
+a latency value for a given delay bdev.
+
 ## v19.07:
 
 ### ftl

doc/jsonrpc.md

@@ -1801,6 +1801,48 @@ Example response:
 }
 ~~~
 
+## bdev_delay_update_latency {#rpc_bdev_delay_update_latency}
+
+Update a target latency value associated with a given delay bdev. Any currently
+outstanding I/O will be completed with the old latency.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+delay_bdev_name         | Required | string      | Name of the delay bdev
+latency_type            | Required | string      | One of: avg_read, avg_write, p99_read, p99_write
+latency_us              | Required | number      | The new latency value in microseconds
+
+### Result
+
+Name of newly created bdev.
+
+### Example
+
+Example request:
+
+~~~
+{
+  "params": {
+    "delay_bdev_name": "Delay0",
+    "latency_type": "avg_read",
+    "latency_us": "100",
+  },
+  "jsonrpc": "2.0",
+  "method": "bdev_delay_update_latency",
+  "id": 1
+}
+~~~
+
+Example response:
+
+~~~
+{
+  "result": "true"
+}
+~~~
+
 ## construct_error_bdev {#rpc_construct_error_bdev}
 
 Construct error bdev.

lib/bdev/delay/vbdev_delay.c

@@ -163,7 +163,13 @@ _process_io_stailq(void *arg, uint64_t ticks)
 			STAILQ_REMOVE(head, io_ctx, delay_bdev_io, link);
 			spdk_bdev_io_complete(SPDK_CONTAINEROF(io_ctx, struct spdk_bdev_io, driver_ctx), io_ctx->status);
 		} else {
-			/* We can assume that I/O are strictly ordered. If one is not expired, we can assume that all after it aren't either. */
+			/* In the general case, I/O will become ready in an fifo order. When timeouts are dynamically
+			 * changed, this is not necessarily the case. However, the normal behavior will be restored
+			 * after the outstanding I/O at the time of the change have been completed.
+			 * This essentially means that moving from a high to low latency creates a dam for the new I/O
+			 * submitted after the latency change. This is considered desirable behavior for the use case where
+			 * we are trying to trigger a pre-defined timeout on an initiator.
+			 */
 			break;
 		}
 	}
@@ -490,6 +496,42 @@ vbdev_delay_insert_association(const char *bdev_name, const char *vbdev_name,
 	return 0;
 }
 
+int
+vbdev_delay_update_latency_value(char *delay_name, uint64_t latency_us, enum delay_io_type type)
+{
+	struct spdk_bdev *delay_bdev;
+	struct vbdev_delay *delay_node;
+	uint64_t ticks_mhz = spdk_get_ticks_hz() / SPDK_SEC_TO_USEC;
+
+	delay_bdev = spdk_bdev_get_by_name(delay_name);
+	if (delay_bdev == NULL) {
+		return -ENODEV;
+	} else if (delay_bdev->module != &delay_if) {
+		return -EINVAL;
+	}
+
+	delay_node = SPDK_CONTAINEROF(delay_bdev, struct vbdev_delay, delay_bdev);
+
+	switch (type) {
+	case DELAY_AVG_READ:
+		delay_node->average_read_latency_ticks = ticks_mhz * latency_us;
+		break;
+	case DELAY_AVG_WRITE:
+		delay_node->average_write_latency_ticks = ticks_mhz * latency_us;
+		break;
+	case DELAY_P99_READ:
+		delay_node->p99_read_latency_ticks = ticks_mhz * latency_us;
+		break;
+	case DELAY_P99_WRITE:
+		delay_node->p99_write_latency_ticks = ticks_mhz * latency_us;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int
 vbdev_delay_init(void)
 {

lib/bdev/delay/vbdev_delay.h

@@ -71,4 +71,15 @@ int create_delay_disk(const char *bdev_name, const char *vbdev_name, uint64_t av
 void delete_delay_disk(struct spdk_bdev *bdev, spdk_bdev_unregister_cb cb_fn,
 		       void *cb_arg);
 
+/**
+ * Update one of the latency values for a given delay bdev.
+ *
+ * \param delay_name The name of the delay bdev
+ * \param latency_us The new latency value, in microseconds
+ * \param type a valid value from the delay_io_type enum
+ * \return 0 on success, -ENODEV if the bdev cannot be found, and -EINVAL if the bdev is not a delay device.
+ */
+int vbdev_delay_update_latency_value(char *delay_name, uint64_t latency_us,
+				     enum delay_io_type type);
+
 #endif /* SPDK_VBDEV_DELAY_H */

lib/bdev/delay/vbdev_delay_rpc.c

@@ -38,6 +38,83 @@
 #include "spdk/string.h"
 #include "spdk_internal/log.h"
 
+struct rpc_update_latency {
+	char *delay_bdev_name;
+	char *latency_type;
+	uint64_t latency_us;
+};
+
+static const struct spdk_json_object_decoder rpc_update_latency_decoders[] = {
+	{"delay_bdev_name", offsetof(struct rpc_update_latency, delay_bdev_name), spdk_json_decode_string},
+	{"latency_type", offsetof(struct rpc_update_latency, latency_type), spdk_json_decode_string},
+	{"latency_us", offsetof(struct rpc_update_latency, latency_us), spdk_json_decode_uint64}
+};
+
+static void
+free_rpc_update_latency(struct rpc_update_latency *req)
+{
+	free(req->delay_bdev_name);
+	free(req->latency_type);
+}
+
+static void
+spdk_rpc_bdev_delay_update_latency(struct spdk_jsonrpc_request *request,
+				   const struct spdk_json_val *params)
+{
+	struct rpc_update_latency req = {NULL};
+	struct spdk_json_write_ctx *w;
+	enum delay_io_type latency_type;
+	int rc = 0;
+
+	if (spdk_json_decode_object(params, rpc_update_latency_decoders,
+				    SPDK_COUNTOF(rpc_update_latency_decoders),
+				    &req)) {
+		SPDK_DEBUGLOG(SPDK_LOG_VBDEV_DELAY, "spdk_json_decode_object failed\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "spdk_json_decode_object failed");
+		goto cleanup;
+	}
+
+	if (!strncmp(req.latency_type, "avg_read", 9)) {
+		latency_type = DELAY_AVG_READ;
+	} else if (!strncmp(req.latency_type, "p99_read", 9)) {
+		latency_type = DELAY_P99_READ;
+	} else if (!strncmp(req.latency_type, "avg_write", 10)) {
+		latency_type = DELAY_AVG_WRITE;
+	} else if (!strncmp(req.latency_type, "p99_write", 10)) {
+		latency_type = DELAY_P99_WRITE;
+	} else {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "Please specify a valid latency type.");
+		goto cleanup;
+	}
+
+	rc = vbdev_delay_update_latency_value(req.delay_bdev_name, req.latency_us, latency_type);
+
+	if (rc == -ENODEV) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "The requested bdev does not exist.");
+		goto cleanup;
+	} else if (rc == -EINVAL) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_REQUEST,
+						 "The requested bdev is not a delay bdev.");
+		goto cleanup;
+	} else if (rc) {
+		/* currently, only the two error cases are defined. Any new error paths should be handled here. */
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "An unknown error occured.");
+		goto cleanup;
+	}
+
+	w = spdk_jsonrpc_begin_result(request);
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+
+cleanup:
+	free_rpc_update_latency(&req);
+}
+SPDK_RPC_REGISTER("bdev_delay_update_latency", spdk_rpc_bdev_delay_update_latency, SPDK_RPC_RUNTIME)
+
 struct rpc_construct_delay {
 	char *base_bdev_name;
 	char *name;

scripts/rpc.py

@@ -446,6 +446,18 @@ if __name__ == "__main__":
     p.add_argument('name', help='delay bdev name')
     p.set_defaults(func=bdev_delay_delete)
 
+    def bdev_delay_update_latency(args):
+        print_json(rpc.bdev.bdev_delay_update_latency(args.client,
+                                                      delay_bdev_name=args.delay_bdev_name,
+                                                      latency_type=args.latency_type,
+                                                      latency_us=args.latency_us))
+    p = subparsers.add_parser('bdev_delay_update_latency',
+                              help='Update one of the latency values for a given delay bdev')
+    p.add_argument('delay_bdev_name', help='The name of the given delay bdev')
+    p.add_argument('latency_type', help='one of: avg_read, avg_write, p99_read, p99_write. No other values accepted.')
+    p.add_argument('latency_us', help='new latency value.', type=int)
+    p.set_defaults(func=bdev_delay_update_latency)
+
     def construct_error_bdev(args):
         print_json(rpc.bdev.construct_error_bdev(args.client,
                                                  base_name=args.base_name))

scripts/rpc/bdev.py

@@ -473,6 +473,25 @@ def bdev_delay_delete(client, name):
     return client.call('bdev_delay_delete', params)
 
 
+def bdev_delay_update_latency(client, delay_bdev_name, latency_type, latency_us):
+    """Update the latency value for a delay block device
+
+    Args:
+        delay_bdev_name: name of the delay bdev
+        latency_type: 'one of: avg_read, avg_write, p99_read, p99_write. No other values accepted.'
+        latency_us: 'new latency value.'
+
+    Returns:
+        True if successful, or a specific error otherwise.
+    """
+    params = {
+        'delay_bdev_name': delay_bdev_name,
+        'latency_type': latency_type,
+        'latency_us': latency_us,
+    }
+    return client.call('bdev_delay_update_latency', params)
+
+
 def delete_error_bdev(client, name):
     """Remove error bdev from the system.