test/nvmf: add application for target fuzz testing.

Enables us to test randomized data against the NVMe-oF target interface.

Change-Id: Ie7ab46949ccd89f74b10b79a24256aeae2df89ab
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/431571
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Hailiang Wang <hailiangx.e.wang@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: qun wan <qun.wan@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>

test/app/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y += bdev_svc histogram_perf jsoncat stub
+DIRS-y += bdev_svc fuzz histogram_perf jsoncat stub
 
 .PHONY: all clean $(DIRS-y)
 

test/app/fuzz/Makefile

@@ -0,0 +1,44 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += nvme_fuzz
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

test/app/fuzz/common/fuzz_common.h

@@ -0,0 +1,303 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/env.h"
+#include "spdk/file.h"
+#include "spdk/base64.h"
+#include "spdk/json.h"
+
+#define DEFAULT_RUNTIME 30 /* seconds */
+#define MAX_RUNTIME_S 86400 /* 24 hours */
+#define IO_TIMEOUT_S 5
+
+#define UNSIGNED_2BIT_MAX ((1 << 2) - 1)
+#define UNSIGNED_4BIT_MAX ((1 << 4) - 1)
+#define UNSIGNED_8BIT_MAX ((1 << 8) - 1)
+
+typedef bool (*json_parse_fn)(void *ele, struct spdk_json_val *val, size_t num_vals);
+
+static void
+fuzz_fill_random_bytes(char *character_repr, size_t len, unsigned int *rand_seed)
+{
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		character_repr[i] = rand_r(rand_seed) % UINT8_MAX;
+	}
+}
+
+static uint64_t
+fuzz_refresh_timeout(void)
+{
+	uint64_t current_ticks;
+	uint64_t new_timeout_ticks;
+
+	current_ticks = spdk_get_ticks();
+
+	new_timeout_ticks = current_ticks + IO_TIMEOUT_S * spdk_get_ticks_hz();
+	assert(new_timeout_ticks > current_ticks);
+
+	return new_timeout_ticks;
+}
+
+static char *
+fuzz_get_value_base_64_buffer(void *item, size_t len)
+{
+	char *value_string;
+	size_t total_size;
+	int rc;
+
+	/* Null pointer */
+	total_size = spdk_base64_get_encoded_strlen(len) + 1;
+
+	value_string = calloc(1, total_size);
+	if (value_string == NULL) {
+		return NULL;
+	}
+
+	rc = spdk_base64_encode(value_string, item, len);
+	if (rc < 0) {
+		free(value_string);
+		return NULL;
+	}
+
+	return value_string;
+}
+
+static int
+fuzz_get_base_64_buffer_value(void *item, size_t len, char *buf, size_t buf_len)
+{
+	size_t size_of_data;
+	char *new_buf;
+	int rc;
+
+	new_buf = malloc(buf_len + 1);
+	if (new_buf == NULL) {
+		return -ENOMEM;
+	}
+
+	snprintf(new_buf, buf_len + 1, "%s", buf);
+
+	size_of_data = spdk_base64_get_decoded_len(buf_len);
+
+	if (size_of_data < len) {
+		free(new_buf);
+		return -EINVAL;
+	}
+
+	rc = spdk_base64_decode(item, &size_of_data, new_buf);
+
+	if (rc || size_of_data != len) {
+		free(new_buf);
+		return -EINVAL;
+	}
+
+	free(new_buf);
+	return 0;
+}
+
+static ssize_t
+read_json_into_buffer(const char *filename, struct spdk_json_val **values, void **file_data)
+{
+	FILE *file = fopen(filename, "r");
+	size_t file_data_size;
+	ssize_t num_json_values = 0, rc;
+
+	if (file == NULL) {
+		/* errno is set by fopen */
+		return 0;
+	}
+
+	*file_data = spdk_posix_file_load(file, &file_data_size);
+	if (*file_data == NULL) {
+		fclose(file);
+		return 0;
+	}
+
+	fclose(file);
+
+	num_json_values = spdk_json_parse(*file_data, file_data_size, NULL, 0, NULL,
+					  SPDK_JSON_PARSE_FLAG_ALLOW_COMMENTS);
+
+	*values = calloc(num_json_values, sizeof(**values));
+	if (values == NULL) {
+		free(*file_data);
+		*file_data = NULL;
+		return 0;
+	}
+
+	rc = spdk_json_parse(*file_data, file_data_size, *values, num_json_values, NULL,
+			     SPDK_JSON_PARSE_FLAG_ALLOW_COMMENTS);
+	if (num_json_values != rc) {
+		free(*values);
+		*values = NULL;
+		free(*file_data);
+		*file_data = NULL;
+		return 0;
+	}
+
+	return num_json_values;
+}
+
+static size_t
+double_arr_size(void **buffer, size_t num_ele, size_t ele_size)
+{
+	void *tmp;
+	size_t new_num_ele, allocation_size;
+
+	if (num_ele > SIZE_MAX / 2) {
+		return 0;
+	}
+
+	new_num_ele = num_ele * 2;
+
+	if (new_num_ele > SIZE_MAX / ele_size) {
+		return 0;
+	}
+
+	allocation_size = new_num_ele * ele_size;
+
+	tmp = realloc(*buffer, allocation_size);
+	if (tmp != NULL) {
+		*buffer = tmp;
+		return new_num_ele;
+	}
+
+	return 0;
+}
+
+static uint64_t
+fuzz_parse_args_into_array(const char *file, void **arr, size_t ele_size, const char *obj_name,
+			   json_parse_fn cb_fn)
+{
+	ssize_t i, num_json_values;
+	struct spdk_json_val *values = NULL, *values_head = NULL, *obj_start;
+	void *file_data = NULL;;
+	char *arr_idx_pointer;
+	size_t num_arr_elements, arr_elements_used, values_in_obj;
+	bool rc;
+
+	num_json_values = read_json_into_buffer(file, &values_head, &file_data);
+	values = values_head;
+	if (num_json_values == 0 || values == NULL) {
+		if (file_data != NULL) {
+			free(file_data);
+		}
+		fprintf(stderr, "The file provided does not exist or we were unable to parse it.\n");
+		return 0;
+	}
+
+	num_arr_elements = 10;
+	arr_elements_used = 0;
+	*arr = calloc(num_arr_elements, ele_size);
+	arr_idx_pointer = (char *)*arr;
+	if (arr_idx_pointer == NULL) {
+		free(values);
+		free(file_data);
+		return 0;
+	}
+
+	i = 0;
+	while (i < num_json_values) {
+		if (values->type != SPDK_JSON_VAL_NAME) {
+			i++;
+			values++;
+			continue;
+		}
+
+		if (!strncmp(values->start, obj_name, values->len)) {
+			i++;
+			values++;
+			assert(values->type == SPDK_JSON_VAL_OBJECT_BEGIN);
+			obj_start = values;
+			values_in_obj = spdk_json_val_len(obj_start);
+			values += values_in_obj;
+			i += values_in_obj;
+
+			rc = cb_fn((void *)arr_idx_pointer, obj_start, values_in_obj);
+			if (rc == false) {
+				fprintf(stderr, "failed to parse file after %lu elements.\n", arr_elements_used);
+				goto fail;
+			}
+
+			arr_idx_pointer += ele_size;
+			arr_elements_used++;
+			if (arr_elements_used == num_arr_elements) {
+				num_arr_elements = double_arr_size(arr, num_arr_elements, ele_size);
+				if (num_arr_elements == 0) {
+					fprintf(stderr, "failed to allocate enough space for all json elements in your file.\n");
+					goto fail;
+				} else {
+					/* reset the array element position in case the pointer changed. */
+					arr_idx_pointer = ((char *)*arr) + arr_elements_used * ele_size;
+				}
+			}
+
+			continue;
+		} else {
+			i++;
+			values++;
+			continue;
+		}
+	}
+
+	if (arr_elements_used == 0) {
+		goto fail;
+	}
+
+	free(values_head);
+	free(file_data);
+	return arr_elements_used;
+fail:
+	free(values_head);
+	free(file_data);
+	free(*arr);
+	*arr = NULL;
+	return 0;
+}
+
+static int
+fuzz_parse_json_num(struct spdk_json_val *val, uint64_t max_val, uint64_t *val_ptr)
+{
+	uint64_t tmp_val;
+	int rc;
+
+	rc = spdk_json_number_to_uint64(val, &tmp_val);
+	if (rc || tmp_val > max_val) {
+		return -EINVAL;
+	} else {
+		*val_ptr = tmp_val;
+		return 0;
+	}
+}

test/app/fuzz/nvme_fuzz/.gitignore

@@ -0,0 +1 @@
+nvme_fuzz

test/app/fuzz/nvme_fuzz/Makefile

@@ -0,0 +1,45 @@
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = nvme_fuzz
+
+CFLAGS += -I$(SPDK_ROOT_DIR)/test/app/fuzz/common
+
+C_SRCS := nvme_fuzz.c
+
+SPDK_LIB_LIST += $(SOCK_MODULES_LIST)
+SPDK_LIB_LIST += conf event json jsonrpc log nvme rpc sock thread trace util
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

test/app/fuzz/nvme_fuzz/README.md

@@ -0,0 +1,49 @@
+# Overview
+
+This application is intended to fuzz test the NVMe-oF target or a physical NVMe drive by
+submitting randomized NVMe commands through the SPDK NVMe initiator. Both local and remote
+drives are configured through a .ini style config file (See the -C option on the application).
+Multiple controllers and namespaces can be exposed to the fuzzer at a time. In order to
+handle multiple namespaces, the fuzzer will round robin assign a thread to each namespace and
+submit commands to that thread at a set queue depth. (currently 128 for I/O, 16 for Admin). The
+application will terminate under three conditions:
+1. The user specified run time expires (see the -t flag).
+2. One of the target controllers stops completing I/O operations back to the fuzzer i.e. controller timeout.
+3. The user specified a json file containing operations to run and the fuzzer has received valid completions for all of them.
+
+# Output
+
+By default, the fuzzer will print commands that:
+1. Complete successfully back from the target, or
+2. Are outstanding at the time of a controller timeout.
+Commands are dumped as named objects in json format which can then be supplied back to the
+script for targeted debugging on a subsequent run. See `Debugging` below.
+By default no output is generated when a specific command is returned with a failed status.
+This can be overridden with the -v flag. if -v is specified, each command will be dumped as
+it is completed in the JSON format specified above.
+At the end of each test run, a summary is printed for each namespace in the following format:
+
+~~~
+NS: 0x200079262300 admin qp, Total commands completed: 462459, total successful commands: 1960, random_seed: 4276918833
+~~~
+
+# Debugging
+
+If a controller hangs when processing I/O generated by the fuzzer, the fuzzer will stop
+submitting I/O and dump out all outstanding I/O on the qpair that timed out. The I/O are
+dumped as valid json. You can combine the dumped commands from the fuzzer into a json
+array in a file and then pass them to the fuzzer using the -j option. Please see the
+example.json file in this directory for an example of a properly formed array of command
+structures.
+
+Please note that you can also craft your own custom command values by using the output
+from the fuzzer as a template.
+
+# JSON Format
+
+Most of the variables in the spdk_nvme_cmd structure are represented as numbers in JSON.
+The only exception to this rule is the dptr union. This is a 16 byte union structure that
+is represented as a base64 string. If writing custom commands for input, please note this
+distinction or the application will be unable to load your custom input.
+
+Happy Fuzzing!

test/app/fuzz/nvme_fuzz/example.json

@@ -0,0 +1,290 @@
+{
+"struct spdk_nvme_cmd": {
+  "opc": 7,
+  "fuse": 1,
+  "rsvd1": 13,
+  "psdt": 1,
+  "cid": 56732,
+  "nsid": 1,
+  "rsvd2": 1516848792,
+  "rsvd3": 1233945838,
+  "mptr": 3452736735,
+  "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+  "cdw10": 3190735246,
+  "cdw11": 2629178873,
+  "cdw12": 138580308,
+  "cdw13": 1603605200,
+  "cdw14": 3031880384,
+  "cdw15": 644909208
+  },
+  "struct spdk_nvme_cmd": {
+  "opc": 24,
+  "fuse": 1,
+  "rsvd1": 13,
+  "psdt": 1,
+  "cid": 56732,
+  "nsid": 1,
+  "rsvd2": 1516848792,
+  "rsvd3": 1233945838,
+  "mptr": 3452736735,
+  "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+  "cdw10": 3190735246,
+  "cdw11": 2629178873,
+  "cdw12": 138580308,
+  "cdw13": 1603605200,
+  "cdw14": 3031880384,
+  "cdw15": 644909208
+  },
+  "struct spdk_nvme_cmd": {
+  "opc": 43,
+  "fuse": 1,
+  "rsvd1": 13,
+  "psdt": 1,
+  "cid": 56732,
+  "nsid": 1,
+  "rsvd2": 1516848792,
+  "rsvd3": 1233945838,
+  "mptr": 3452736735,
+  "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+  "cdw10": 3190735246,
+  "cdw11": 2629178873,
+  "cdw12": 138580308,
+  "cdw13": 1603605200,
+  "cdw14": 3031880384,
+  "cdw15": 644909208
+  },
+  "struct spdk_nvme_cmd": {
+  "opc": 12,
+  "fuse": 1,
+  "rsvd1": 13,
+  "psdt": 1,
+  "cid": 56732,
+  "nsid": 1,
+  "rsvd2": 1516848792,
+  "rsvd3": 1233945838,
+  "mptr": 3452736735,
+  "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+  "cdw10": 3190735246,
+  "cdw11": 2629178873,
+  "cdw12": 138580308,
+  "cdw13": 1603605200,
+  "cdw14": 3031880384,
+  "cdw15": 644909208
+  },
+  "struct spdk_nvme_cmd": {
+    "opc": 7,
+    "fuse": 1,
+    "rsvd1": 13,
+    "psdt": 1,
+    "cid": 56732,
+    "nsid": 1,
+    "rsvd2": 1516848792,
+    "rsvd3": 1233945838,
+    "mptr": 3452736735,
+    "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+    "cdw10": 3190735246,
+    "cdw11": 2629178873,
+    "cdw12": 138580308,
+    "cdw13": 1603605200,
+    "cdw14": 3031880384,
+    "cdw15": 644909208
+    },
+    "struct spdk_nvme_cmd": {
+    "opc": 24,
+    "fuse": 1,
+    "rsvd1": 13,
+    "psdt": 1,
+    "cid": 56732,
+    "nsid": 1,
+    "rsvd2": 1516848792,
+    "rsvd3": 1233945838,
+    "mptr": 3452736735,
+    "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+    "cdw10": 3190735246,
+    "cdw11": 2629178873,
+    "cdw12": 138580308,
+    "cdw13": 1603605200,
+    "cdw14": 3031880384,
+    "cdw15": 644909208
+    },
+    "struct spdk_nvme_cmd": {
+    "opc": 43,
+    "fuse": 1,
+    "rsvd1": 13,
+    "psdt": 1,
+    "cid": 56732,
+    "nsid": 1,
+    "rsvd2": 1516848792,
+    "rsvd3": 1233945838,
+    "mptr": 3452736735,
+    "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+    "cdw10": 3190735246,
+    "cdw11": 2629178873,
+    "cdw12": 138580308,
+    "cdw13": 1603605200,
+    "cdw14": 3031880384,
+    "cdw15": 644909208
+    },
+    "struct spdk_nvme_cmd": {
+    "opc": 12,
+    "fuse": 1,
+    "rsvd1": 13,
+    "psdt": 1,
+    "cid": 56732,
+    "nsid": 1,
+    "rsvd2": 1516848792,
+    "rsvd3": 1233945838,
+    "mptr": 3452736735,
+    "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+    "cdw10": 3190735246,
+    "cdw11": 2629178873,
+    "cdw12": 138580308,
+    "cdw13": 1603605200,
+    "cdw14": 3031880384,
+    "cdw15": 644909208
+    },
+    "struct spdk_nvme_cmd": {
+      "opc": 7,
+      "fuse": 1,
+      "rsvd1": 13,
+      "psdt": 1,
+      "cid": 56732,
+      "nsid": 1,
+      "rsvd2": 1516848792,
+      "rsvd3": 1233945838,
+      "mptr": 3452736735,
+      "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+      "cdw10": 3190735246,
+      "cdw11": 2629178873,
+      "cdw12": 138580308,
+      "cdw13": 1603605200,
+      "cdw14": 3031880384,
+      "cdw15": 644909208
+      },
+      "struct spdk_nvme_cmd": {
+      "opc": 24,
+      "fuse": 1,
+      "rsvd1": 13,
+      "psdt": 1,
+      "cid": 56732,
+      "nsid": 1,
+      "rsvd2": 1516848792,
+      "rsvd3": 1233945838,
+      "mptr": 3452736735,
+      "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+      "cdw10": 3190735246,
+      "cdw11": 2629178873,
+      "cdw12": 138580308,
+      "cdw13": 1603605200,
+      "cdw14": 3031880384,
+      "cdw15": 644909208
+      },
+      "struct spdk_nvme_cmd": {
+      "opc": 43,
+      "fuse": 1,
+      "rsvd1": 13,
+      "psdt": 1,
+      "cid": 56732,
+      "nsid": 1,
+      "rsvd2": 1516848792,
+      "rsvd3": 1233945838,
+      "mptr": 3452736735,
+      "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+      "cdw10": 3190735246,
+      "cdw11": 2629178873,
+      "cdw12": 138580308,
+      "cdw13": 1603605200,
+      "cdw14": 3031880384,
+      "cdw15": 644909208
+      },
+      "struct spdk_nvme_cmd": {
+      "opc": 12,
+      "fuse": 1,
+      "rsvd1": 13,
+      "psdt": 1,
+      "cid": 56732,
+      "nsid": 1,
+      "rsvd2": 1516848792,
+      "rsvd3": 1233945838,
+      "mptr": 3452736735,
+      "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+      "cdw10": 3190735246,
+      "cdw11": 2629178873,
+      "cdw12": 138580308,
+      "cdw13": 1603605200,
+      "cdw14": 3031880384,
+      "cdw15": 644909208
+      },
+      "struct spdk_nvme_cmd": {
+        "opc": 7,
+        "fuse": 1,
+        "rsvd1": 13,
+        "psdt": 1,
+        "cid": 56732,
+        "nsid": 1,
+        "rsvd2": 1516848792,
+        "rsvd3": 1233945838,
+        "mptr": 3452736735,
+        "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+        "cdw10": 3190735246,
+        "cdw11": 2629178873,
+        "cdw12": 138580308,
+        "cdw13": 1603605200,
+        "cdw14": 3031880384,
+        "cdw15": 644909208
+        },
+        "struct spdk_nvme_cmd": {
+        "opc": 24,
+        "fuse": 1,
+        "rsvd1": 13,
+        "psdt": 1,
+        "cid": 56732,
+        "nsid": 1,
+        "rsvd2": 1516848792,
+        "rsvd3": 1233945838,
+        "mptr": 3452736735,
+        "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+        "cdw10": 3190735246,
+        "cdw11": 2629178873,
+        "cdw12": 138580308,
+        "cdw13": 1603605200,
+        "cdw14": 3031880384,
+        "cdw15": 644909208
+        },
+        "struct spdk_nvme_cmd": {
+        "opc": 43,
+        "fuse": 1,
+        "rsvd1": 13,
+        "psdt": 1,
+        "cid": 56732,
+        "nsid": 1,
+        "rsvd2": 1516848792,
+        "rsvd3": 1233945838,
+        "mptr": 3452736735,
+        "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+        "cdw10": 3190735246,
+        "cdw11": 2629178873,
+        "cdw12": 138580308,
+        "cdw13": 1603605200,
+        "cdw14": 3031880384,
+        "cdw15": 644909208
+        },
+        "struct spdk_nvme_cmd": {
+        "opc": 12,
+        "fuse": 1,
+        "rsvd1": 13,
+        "psdt": 1,
+        "cid": 56732,
+        "nsid": 1,
+        "rsvd2": 1516848792,
+        "rsvd3": 1233945838,
+        "mptr": 3452736735,
+        "dptr": "FHENPcH+xM0tioD+0SrNrQ==",
+        "cdw10": 3190735246,
+        "cdw11": 2629178873,
+        "cdw12": 138580308,
+        "cdw13": 1603605200,
+        "cdw14": 3031880384,
+        "cdw15": 644909208
+        }
+}

test/app/fuzz/nvme_fuzz/nvme_fuzz.c

@@ -0,0 +1,929 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/conf.h"
+#include "spdk/env.h"
+#include "spdk/event.h"
+#include "spdk/util.h"
+#include "spdk/string.h"
+#include "spdk/nvme_spec.h"
+#include "spdk/nvme.h"
+#include "spdk/likely.h"
+#include "spdk/json.h"
+#include "fuzz_common.h"
+
+#define UNIQUE_OPCODES 256
+
+const char g_nvme_cmd_json_name[] = "struct spdk_nvme_cmd";
+char *g_conf_file;
+char *g_json_file = NULL;
+uint64_t g_runtime_ticks;
+unsigned int g_seed_value = 0;
+int g_runtime;
+
+int g_num_active_threads = 0;
+uint32_t g_admin_depth = 16;
+uint32_t g_io_depth = 128;
+
+bool g_valid_ns_only = false;
+bool g_verbose_mode = false;
+bool g_run_admin_commands = false;
+bool g_run;
+
+struct spdk_poller *g_app_completion_poller;
+bool g_successful_io_opcodes[UNIQUE_OPCODES] = {0};
+bool g_successful_admin_opcodes[UNIQUE_OPCODES] = {0};
+
+struct spdk_nvme_cmd *g_cmd_array;
+size_t g_cmd_array_size;
+
+/* I need context objects here because I need to keep track of all I/O that are in flight. */
+struct nvme_fuzz_request {
+	struct spdk_nvme_cmd		cmd;
+	struct nvme_fuzz_qp		*qp;
+	TAILQ_ENTRY(nvme_fuzz_request)	link;
+};
+
+struct nvme_fuzz_trid {
+	struct spdk_nvme_transport_id	trid;
+	TAILQ_ENTRY(nvme_fuzz_trid)	tailq;
+};
+
+struct nvme_fuzz_ctrlr {
+	struct spdk_nvme_ctrlr		*ctrlr;
+	TAILQ_ENTRY(nvme_fuzz_ctrlr)	tailq;
+};
+
+struct nvme_fuzz_qp {
+	struct spdk_nvme_qpair		*qpair;
+	/* array of context objects equal in length to the queue depth */
+	struct nvme_fuzz_request	*req_ctx;
+	TAILQ_HEAD(, nvme_fuzz_request)	free_ctx_objs;
+	TAILQ_HEAD(, nvme_fuzz_request)	outstanding_ctx_objs;
+	unsigned int			random_seed;
+	uint64_t			completed_cmd_counter;
+	uint64_t			submitted_cmd_counter;
+	uint64_t			successful_completed_cmd_counter;
+	uint64_t			timeout_tsc;
+	uint32_t			num_cmds_outstanding;
+	bool				timed_out;
+	bool				is_admin;
+};
+
+struct nvme_fuzz_ns {
+	struct spdk_nvme_ns		*ns;
+	struct spdk_nvme_ctrlr		*ctrlr;
+	struct spdk_thread		*thread;
+	struct spdk_poller		*req_poller;
+	struct nvme_fuzz_qp		io_qp;
+	struct nvme_fuzz_qp		a_qp;
+	uint32_t			nsid;
+	TAILQ_ENTRY(nvme_fuzz_ns)	tailq;
+};
+
+static TAILQ_HEAD(, nvme_fuzz_ns) g_ns_list = TAILQ_HEAD_INITIALIZER(g_ns_list);
+static TAILQ_HEAD(, nvme_fuzz_ctrlr) g_ctrlr_list = TAILQ_HEAD_INITIALIZER(g_ctrlr_list);
+static TAILQ_HEAD(, nvme_fuzz_trid) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list);
+
+static bool
+parse_nvme_cmd_obj(void *item, struct spdk_json_val *value, size_t num_values)
+{
+	struct spdk_nvme_cmd *cmd = item;
+	struct spdk_json_val *next_val;
+	uint64_t tmp_val;
+	size_t i = 0;
+
+	while (i < num_values) {
+		if (value->type == SPDK_JSON_VAL_NAME) {
+			next_val = value + 1;
+			if (!strncmp(value->start, "opc", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UNSIGNED_8BIT_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->opc = tmp_val;
+				}
+			} else if (!strncmp(value->start, "fuse", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UNSIGNED_2BIT_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->fuse = tmp_val;
+				}
+			} else if (!strncmp(value->start, "rsvd1", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UNSIGNED_4BIT_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->rsvd1 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "psdt", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UNSIGNED_2BIT_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->psdt = tmp_val;
+				}
+			} else if (!strncmp(value->start, "cid", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT16_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->cid = tmp_val;
+				}
+			} else if (!strncmp(value->start, "nsid", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->nsid = tmp_val;
+				}
+			} else if (!strncmp(value->start, "rsvd2", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->rsvd2 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "rsvd3", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->rsvd3 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "mptr", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT64_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->mptr = tmp_val;
+				}
+			} else if (!strncmp(value->start, "dptr", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_STRING) {
+					if (fuzz_get_base_64_buffer_value(&cmd->dptr, sizeof(cmd->dptr), (char *)next_val->start,
+									  next_val->len)) {
+						goto invalid;
+					}
+				}
+			} else if (!strncmp(value->start, "cdw10", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->cdw10 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "cdw11", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->cdw11 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "cdw12", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->cdw12 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "cdw13", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->cdw13 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "cdw14", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->cdw14 = tmp_val;
+				}
+			} else if (!strncmp(value->start, "cdw15", value->len)) {
+				if (next_val->type == SPDK_JSON_VAL_NUMBER) {
+					if (fuzz_parse_json_num(next_val, UINT32_MAX, &tmp_val)) {
+						goto invalid;
+					}
+					cmd->cdw15 = tmp_val;
+				}
+			}
+		}
+		i++;
+		value++;
+	}
+	return true;
+
+invalid:
+	fprintf(stderr, "Invalid value supplied for cmd->%.*s: %.*s\n", value->len, (char *)value->start,
+		next_val->len, (char *)next_val->start);
+	return false;
+}
+
+static void
+report_successful_opcodes(bool *array, int length)
+{
+	int i;
+
+	for (i = 0; i < length; i++) {
+		if (array[i] == true) {
+			printf("%d, ", i);
+		}
+	}
+	printf("\n");
+}
+
+static int
+print_nvme_cmd(void *cb_ctx, const void *data, size_t size)
+{
+	fprintf(stderr, "%s\n", (const char *)data);
+	return 0;
+}
+
+static void
+json_dump_nvme_cmd(struct spdk_nvme_cmd *cmd)
+{
+	struct spdk_json_write_ctx *w;
+	char *dptr_value;
+
+	dptr_value = fuzz_get_value_base_64_buffer(&cmd->dptr, sizeof(cmd->dptr));
+	if (dptr_value == NULL) {
+		fprintf(stderr, "Unable to allocate buffer context for printing command.\n");
+		return;
+	}
+
+	w = spdk_json_write_begin(print_nvme_cmd, cmd, SPDK_JSON_WRITE_FLAG_FORMATTED);
+	if (w == NULL) {
+		fprintf(stderr, "Unable to allocate json context for printing command.\n");
+		free(dptr_value);
+		return;
+	}
+
+	spdk_json_write_named_object_begin(w, g_nvme_cmd_json_name);
+	spdk_json_write_named_uint32(w, "opc", cmd->opc);
+	spdk_json_write_named_uint32(w, "fuse", cmd->fuse);
+	spdk_json_write_named_uint32(w, "rsvd1", cmd->rsvd1);
+	spdk_json_write_named_uint32(w, "psdt", cmd->psdt);
+	spdk_json_write_named_uint32(w, "cid", cmd->cid);
+	spdk_json_write_named_uint32(w, "nsid", cmd->nsid);
+	spdk_json_write_named_uint32(w, "rsvd2", cmd->rsvd2);
+	spdk_json_write_named_uint32(w, "rsvd3", cmd->rsvd3);
+	spdk_json_write_named_uint32(w, "mptr", cmd->mptr);
+	spdk_json_write_named_string(w, "dptr", dptr_value);
+	spdk_json_write_named_uint32(w, "cdw10", cmd->cdw10);
+	spdk_json_write_named_uint32(w, "cdw11", cmd->cdw11);
+	spdk_json_write_named_uint32(w, "cdw12", cmd->cdw12);
+	spdk_json_write_named_uint32(w, "cdw13", cmd->cdw13);
+	spdk_json_write_named_uint32(w, "cdw14", cmd->cdw14);
+	spdk_json_write_named_uint32(w, "cdw15", cmd->cdw15);
+	spdk_json_write_object_end(w);
+
+	free(dptr_value);
+	spdk_json_write_end(w);
+}
+
+static void
+json_dump_nvme_cmd_list(struct nvme_fuzz_qp *qp)
+{
+	struct nvme_fuzz_request *ctx;
+
+	TAILQ_FOREACH(ctx, &qp->outstanding_ctx_objs, link) {
+		json_dump_nvme_cmd(&ctx->cmd);
+	}
+}
+
+static void
+handle_timeout(struct nvme_fuzz_qp *qp, bool is_admin)
+{
+	fprintf(stderr, "An %s queue has timed out. Dumping all outstanding commands from that queue\n",
+		is_admin ? "Admin" : "I/O");
+	json_dump_nvme_cmd_list(qp);
+	qp->timed_out = true;
+}
+
+static void submit_ns_cmds(struct nvme_fuzz_ns *ns_entry);
+
+static void
+nvme_fuzz_cpl_cb(void *cb_arg, const struct spdk_nvme_cpl *cpl)
+{
+	struct nvme_fuzz_request *ctx = cb_arg;
+	struct nvme_fuzz_qp *qp = ctx->qp;
+
+	qp->completed_cmd_counter++;
+	if (spdk_unlikely(cpl->status.sc == SPDK_NVME_SC_SUCCESS)) {
+		fprintf(stderr, "The following %s command (command num %lu) completed successfully\n",
+			qp->is_admin ? "Admin" : "I/O", qp->completed_cmd_counter);
+		qp->successful_completed_cmd_counter++;
+		json_dump_nvme_cmd(&ctx->cmd);
+
+		if (qp->is_admin) {
+			__sync_bool_compare_and_swap(&g_successful_admin_opcodes[ctx->cmd.opc], false, true);
+		} else {
+			__sync_bool_compare_and_swap(&g_successful_io_opcodes[ctx->cmd.opc], false, true);
+		}
+	} else if (g_verbose_mode == true) {
+		fprintf(stderr, "The following %s command (command num %lu) failed as expected.\n",
+			qp->is_admin ? "Admin" : "I/O", qp->completed_cmd_counter);
+		json_dump_nvme_cmd(&ctx->cmd);
+	}
+
+	qp->timeout_tsc = fuzz_refresh_timeout();
+	TAILQ_REMOVE(&qp->outstanding_ctx_objs, ctx, link);
+	TAILQ_INSERT_HEAD(&qp->free_ctx_objs, ctx, link);
+	assert(qp->num_cmds_outstanding > 0);
+	qp->num_cmds_outstanding--;
+}
+
+static int
+poll_for_completions(void *arg)
+{
+	struct nvme_fuzz_ns *ns_entry = arg;
+	uint64_t current_ticks = spdk_get_ticks();
+	uint64_t *counter;
+	if (!ns_entry->io_qp.timed_out) {
+		spdk_nvme_qpair_process_completions(ns_entry->io_qp.qpair, 0);
+		/* SAlways have to process admin completions for the purposes of keep alive. */
+		spdk_nvme_ctrlr_process_admin_completions(ns_entry->ctrlr);
+	}
+
+	if (g_cmd_array) {
+		if (g_run_admin_commands) {
+			counter = &ns_entry->a_qp.submitted_cmd_counter;
+		} else {
+			counter = &ns_entry->io_qp.submitted_cmd_counter;
+		}
+
+		if (*counter >= g_cmd_array_size) {
+			g_run = false;
+		}
+	} else {
+		if (current_ticks > g_runtime_ticks) {
+			g_run = false;
+		}
+	}
+
+	if (ns_entry->a_qp.timeout_tsc < current_ticks && !ns_entry->a_qp.timed_out &&
+	    ns_entry->a_qp.num_cmds_outstanding > 0) {
+		handle_timeout(&ns_entry->a_qp, true);
+	}
+
+	if (ns_entry->io_qp.timeout_tsc < current_ticks && !ns_entry->io_qp.timed_out &&
+	    ns_entry->io_qp.num_cmds_outstanding > 0) {
+		handle_timeout(&ns_entry->io_qp, false);
+	}
+
+	submit_ns_cmds(ns_entry);
+
+	if (g_run) {
+		return 0;
+	}
+	/*
+	 * We either processed all I/O properly and can shut down normally, or we
+	 * had a qp time out and we need to exit without reducing the values to 0.
+	 */
+	if (ns_entry->io_qp.num_cmds_outstanding == 0 &&
+	    ns_entry->a_qp.num_cmds_outstanding == 0) {
+		goto exit_handler;
+	} else if (ns_entry->io_qp.timed_out && (!g_run_admin_commands || ns_entry->a_qp.timed_out)) {
+		goto exit_handler;
+	} else {
+		return 0;
+	}
+
+exit_handler:
+	spdk_poller_unregister(&ns_entry->req_poller);
+	__sync_sub_and_fetch(&g_num_active_threads, 1);
+	spdk_thread_exit(ns_entry->thread);
+	return 0;
+}
+
+static void
+prep_nvme_cmd(struct nvme_fuzz_ns *ns_entry, struct nvme_fuzz_qp *qp, struct nvme_fuzz_request *ctx)
+{
+	if (g_cmd_array) {
+		memcpy(&ctx->cmd, &g_cmd_array[qp->submitted_cmd_counter], sizeof(ctx->cmd));
+	} else {
+		fuzz_fill_random_bytes((char *)&ctx->cmd, sizeof(ctx->cmd), &qp->random_seed);
+
+		if (g_valid_ns_only) {
+			ctx->cmd.nsid = ns_entry->nsid;
+		}
+	}
+}
+
+static int
+submit_qp_cmds(struct nvme_fuzz_ns *ns, struct nvme_fuzz_qp *qp)
+{
+	struct nvme_fuzz_request *ctx;
+	int rc;
+
+	if (qp->timed_out) {
+		return 0;
+	}
+	/* If we are reading from an array, we need to stop after the last one. */
+	while ((qp->submitted_cmd_counter < g_cmd_array_size || g_cmd_array_size == 0) &&
+	       !TAILQ_EMPTY(&qp->free_ctx_objs)) {
+		ctx = TAILQ_FIRST(&qp->free_ctx_objs);
+		prep_nvme_cmd(ns, qp, ctx);
+
+		TAILQ_REMOVE(&qp->free_ctx_objs, ctx, link);
+		TAILQ_INSERT_HEAD(&qp->outstanding_ctx_objs, ctx, link);
+		qp->num_cmds_outstanding++;
+		qp->submitted_cmd_counter++;
+		if (qp->is_admin) {
+			rc = spdk_nvme_ctrlr_cmd_admin_raw(ns->ctrlr, &ctx->cmd, NULL, 0, nvme_fuzz_cpl_cb, ctx);
+		} else {
+			rc = spdk_nvme_ctrlr_cmd_io_raw(ns->ctrlr, qp->qpair, &ctx->cmd, NULL, 0, nvme_fuzz_cpl_cb, ctx);
+		}
+		if (rc) {
+			return rc;
+		}
+	}
+	return 0;
+}
+
+static void
+submit_ns_cmds(struct nvme_fuzz_ns *ns_entry)
+{
+	int rc;
+
+	if (!g_run) {
+		return;
+	}
+
+	if (g_run_admin_commands) {
+		rc = submit_qp_cmds(ns_entry, &ns_entry->a_qp);
+		if (rc) {
+			goto err_exit;
+		}
+	}
+
+	if (g_cmd_array == NULL || !g_run_admin_commands) {
+		rc = submit_qp_cmds(ns_entry, &ns_entry->io_qp);
+	}
+err_exit:
+	if (rc) {
+		/*
+		 * I see the prospect of having a broken qpair on one ns as interesting
+		 * enough to recommend stopping the application.
+		 */
+		fprintf(stderr, "Unable to submit command with rc %d\n", rc);
+		g_run = false;
+	}
+}
+
+static void
+free_namespaces(void)
+{
+	struct nvme_fuzz_ns *ns, *tmp;
+
+	TAILQ_FOREACH_SAFE(ns, &g_ns_list, tailq, tmp) {
+		printf("NS: %p I/O qp, Total commands completed: %lu, total successful commands: %lu, random_seed: %u\n",
+		       ns->ns,
+		       ns->io_qp.completed_cmd_counter, ns->io_qp.successful_completed_cmd_counter, ns->io_qp.random_seed);
+		printf("NS: %p admin qp, Total commands completed: %lu, total successful commands: %lu, random_seed: %u\n",
+		       ns->ns,
+		       ns->a_qp.completed_cmd_counter, ns->a_qp.successful_completed_cmd_counter, ns->a_qp.random_seed);
+
+		TAILQ_REMOVE(&g_ns_list, ns, tailq);
+		if (ns->io_qp.qpair) {
+			spdk_nvme_ctrlr_free_io_qpair(ns->io_qp.qpair);
+		}
+		if (ns->io_qp.req_ctx) {
+			free(ns->io_qp.req_ctx);
+		}
+		if (ns->a_qp.req_ctx) {
+			free(ns->a_qp.req_ctx);
+		}
+		free(ns);
+	}
+}
+
+static void
+free_controllers(void)
+{
+	struct nvme_fuzz_ctrlr *ctrlr, *tmp;
+
+	TAILQ_FOREACH_SAFE(ctrlr, &g_ctrlr_list, tailq, tmp) {
+		TAILQ_REMOVE(&g_ctrlr_list, ctrlr, tailq);
+		spdk_nvme_detach(ctrlr->ctrlr);
+		free(ctrlr);
+	}
+}
+
+static void
+free_trids(void)
+{
+	struct nvme_fuzz_trid *trid, *tmp;
+
+	TAILQ_FOREACH_SAFE(trid, &g_trid_list, tailq, tmp) {
+		TAILQ_REMOVE(&g_trid_list, trid, tailq);
+		free(trid);
+	}
+}
+
+static void
+register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, uint32_t nsid)
+{
+	struct nvme_fuzz_ns *ns_entry;
+
+	ns_entry = calloc(1, sizeof(struct nvme_fuzz_ns));
+	if (ns_entry == NULL) {
+		fprintf(stderr, "Unable to allocate an entry for a namespace\n");
+		return;
+	}
+
+	ns_entry->ns = ns;
+	ns_entry->ctrlr = ctrlr;
+	ns_entry->nsid = nsid;
+
+	TAILQ_INIT(&ns_entry->io_qp.free_ctx_objs);
+	TAILQ_INIT(&ns_entry->io_qp.outstanding_ctx_objs);
+	if (g_run_admin_commands) {
+		ns_entry->a_qp.qpair = NULL;
+		TAILQ_INIT(&ns_entry->a_qp.free_ctx_objs);
+		TAILQ_INIT(&ns_entry->a_qp.outstanding_ctx_objs);
+	}
+	TAILQ_INSERT_TAIL(&g_ns_list, ns_entry, tailq);
+}
+
+static void
+register_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_fuzz_ctrlr *ctrlr_entry;
+	uint32_t nsid;
+	struct spdk_nvme_ns *ns;
+
+	ctrlr_entry = calloc(1, sizeof(struct nvme_fuzz_ctrlr));
+	if (ctrlr_entry == NULL) {
+		fprintf(stderr, "Unable to allocate an entry for a controller\n");
+		return;
+	}
+
+	ctrlr_entry->ctrlr = ctrlr;
+	TAILQ_INSERT_TAIL(&g_ctrlr_list, ctrlr_entry, tailq);
+
+	for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); nsid != 0;
+	     nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+		ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+		if (ns == NULL) {
+			continue;
+		}
+		register_ns(ctrlr, ns, nsid);
+	}
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+	  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+	register_ctrlr(ctrlr);
+}
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, struct spdk_nvme_ctrlr_opts *opts)
+{
+	printf("Controller trtype %s\ttraddr %s\n", spdk_nvme_transport_id_trtype_str(trid->trtype),
+	       trid->traddr);
+
+	return true;
+}
+
+static int
+prep_qpair(struct nvme_fuzz_ns *ns, struct nvme_fuzz_qp *qp, uint32_t max_qdepth)
+{
+	uint32_t i;
+
+	/* ensure that each qpair gets a unique random seed for maximum command dispersion. */
+
+	if (g_seed_value != 0) {
+		qp->random_seed = g_seed_value;
+	} else {
+		/* Take the low 32 bits of spdk_get_ticks. This should be more granular than time(). */
+		qp->random_seed = spdk_get_ticks();
+	}
+
+	qp->timeout_tsc = fuzz_refresh_timeout();
+
+	qp->req_ctx = calloc(max_qdepth, sizeof(struct nvme_fuzz_request));
+	if (qp->req_ctx == NULL) {
+		fprintf(stderr, "Unable to allocate I/O contexts for I/O qpair.\n");
+		return -1;
+	}
+
+	for (i = 0; i < max_qdepth; i++) {
+		qp->req_ctx[i].qp = qp;
+		TAILQ_INSERT_HEAD(&qp->free_ctx_objs, &qp->req_ctx[i], link);
+	}
+
+	return 0;
+}
+
+static int
+prepare_qpairs(void)
+{
+	struct spdk_nvme_io_qpair_opts opts;
+	struct nvme_fuzz_ns *ns_entry;
+
+	TAILQ_FOREACH(ns_entry, &g_ns_list, tailq) {
+		spdk_nvme_ctrlr_get_default_io_qpair_opts(ns_entry->ctrlr, &opts, sizeof(opts));
+		ns_entry->io_qp.qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, &opts, sizeof(opts));
+		if (ns_entry->io_qp.qpair == NULL) {
+			fprintf(stderr, "Unable to create a qpair for a namespace\n");
+			return -1;
+		}
+
+		ns_entry->io_qp.is_admin = false;
+		if (prep_qpair(ns_entry, &ns_entry->io_qp, g_io_depth) != 0) {
+			fprintf(stderr, "Unable to allocate request contexts for I/O qpair.\n");
+			return -1;
+		}
+
+		if (g_run_admin_commands) {
+			ns_entry->a_qp.is_admin = true;
+			if (prep_qpair(ns_entry, &ns_entry->a_qp, g_admin_depth) != 0) {
+				fprintf(stderr, "Unable to allocate request contexts for admin qpair.\n");
+				return -1;
+			}
+		}
+	}
+	return 0;
+}
+
+static void
+start_ns_poller(void *ctx)
+{
+	struct nvme_fuzz_ns *ns_entry = ctx;
+
+	ns_entry->req_poller = spdk_poller_register(poll_for_completions, ns_entry, 0);
+	submit_ns_cmds(ns_entry);
+}
+
+static int
+check_app_completion(void *ctx)
+{
+
+	if (g_num_active_threads <= 0) {
+		spdk_poller_unregister(&g_app_completion_poller);
+		if (g_cmd_array) {
+			free(g_cmd_array);
+		}
+		printf("Fuzzing completed. Shutting down the fuzz application\n\n");
+		printf("Dumping successful admin opcodes:\n");
+		report_successful_opcodes(g_successful_admin_opcodes, UNIQUE_OPCODES);
+		printf("Dumping successful io opcodes:\n");
+		report_successful_opcodes(g_successful_io_opcodes, UNIQUE_OPCODES);
+		free_namespaces();
+		free_controllers();
+		free_trids();
+		spdk_app_stop(0);
+	}
+	return 0;
+}
+
+static void
+begin_fuzz(void *ctx)
+{
+	struct nvme_fuzz_ns *ns_entry;
+	struct nvme_fuzz_trid *trid;
+	int rc;
+
+	if (!spdk_iommu_is_enabled()) {
+		/* Don't set rc to an error code here. We don't want to fail an automated test based on this. */
+		fprintf(stderr, "The IOMMU must be enabled to run this program to avoid unsafe memory accesses.\n");
+		rc = 0;
+		goto out;
+	}
+
+	TAILQ_FOREACH(trid, &g_trid_list, tailq) {
+		if (spdk_nvme_probe(&trid->trid, trid, probe_cb, attach_cb, NULL) != 0) {
+			fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n",
+				trid->trid.traddr);
+			rc = -1;
+			goto out;
+		}
+	}
+
+	if (TAILQ_EMPTY(&g_ns_list)) {
+		fprintf(stderr, "No valid NVMe Namespaces to fuzz\n");
+		rc = -EINVAL;
+		goto out;
+	}
+
+	rc = prepare_qpairs();
+
+	if (rc < 0) {
+		fprintf(stderr, "Unable to prepare the qpairs\n");
+		goto out;
+	}
+
+	g_runtime_ticks = spdk_get_ticks() + g_runtime * spdk_get_ticks_hz();
+
+	/* Assigning all of the threads and then starting them makes cleanup easier. */
+	TAILQ_FOREACH(ns_entry, &g_ns_list, tailq) {
+		ns_entry->thread = spdk_thread_create(NULL, NULL);
+		if (ns_entry->thread == NULL) {
+			fprintf(stderr, "Failed to allocate thread for namespace.\n");
+			goto out;
+		}
+	}
+
+	TAILQ_FOREACH(ns_entry, &g_ns_list, tailq) {
+		spdk_thread_send_msg(ns_entry->thread, start_ns_poller, ns_entry);
+		__sync_add_and_fetch(&g_num_active_threads, 1);
+	}
+
+	g_app_completion_poller = spdk_poller_register(check_app_completion, NULL, 1000000);
+	return;
+out:
+	printf("Shutting down the fuzz application\n");
+	free_namespaces();
+	free_controllers();
+	free_trids();
+	spdk_app_stop(rc);
+}
+
+static int
+parse_trids(void)
+{
+	struct spdk_conf *config = NULL;
+	struct spdk_conf_section *sp;
+	const char *trid_char;
+	struct nvme_fuzz_trid *current_trid;
+	int num_subsystems = 0;
+	int rc = 0;
+
+	if (g_conf_file) {
+		config = spdk_conf_allocate();
+		if (!config) {
+			fprintf(stderr, "Unable to allocate an spdk_conf object\n");
+			return -1;
+		}
+
+		rc = spdk_conf_read(config, g_conf_file);
+		if (rc) {
+			fprintf(stderr, "Unable to convert the conf file into a readable system\n");
+			rc = -1;
+			goto exit;
+		}
+
+		sp = spdk_conf_find_section(config, "Nvme");
+
+		if (sp == NULL) {
+			fprintf(stderr, "No Nvme configuration in conf file\n");
+			goto exit;
+		}
+
+		while ((trid_char = spdk_conf_section_get_nmval(sp, "TransportID", num_subsystems, 0)) != NULL) {
+			current_trid = malloc(sizeof(struct nvme_fuzz_trid));
+			if (!current_trid) {
+				fprintf(stderr, "Unable to allocate memory for transport ID\n");
+				rc = -1;
+				goto exit;
+			}
+			rc = spdk_nvme_transport_id_parse(&current_trid->trid, trid_char);
+
+			if (rc < 0) {
+				fprintf(stderr, "failed to parse transport ID: %s\n", trid_char);
+				free(current_trid);
+				rc = -1;
+				goto exit;
+			}
+			TAILQ_INSERT_TAIL(&g_trid_list, current_trid, tailq);
+			num_subsystems++;
+		}
+	}
+
+exit:
+	if (config != NULL) {
+		spdk_conf_free(config);
+	}
+	return rc;
+}
+
+static void
+nvme_fuzz_usage(void)
+{
+	fprintf(stderr, " -a                        Perform admin commands. if -j is specified, \
+only admin commands will run. Otherwise they will be run in tandem with I/O commands.\n");
+	fprintf(stderr, " -C <path>                 Path to a configuration file.\n");
+	fprintf(stderr,
+		" -j <path>                 Path to a json file containing named objects of type spdk_nvme_cmd. If this option is specified, -t will be ignored.\n");
+	fprintf(stderr, " -N                        Target only valid namespace with commands. \
+This helps dig deeper into other errors besides invalid namespace.\n");
+	fprintf(stderr, " -S <integer>              Seed value for test.\n");
+	fprintf(stderr,
+		" -t <integer>              Time in seconds to run the fuzz test. Only valid if -j is not specified.\n");
+	fprintf(stderr, " -v                        Enable logging of each submitted command.\n");
+}
+
+static int
+nvme_fuzz_parse(int ch, char *arg)
+{
+	int64_t error_test;
+
+	switch (ch) {
+	case 'a':
+		g_run_admin_commands = true;
+		break;
+	case 'C':
+		g_conf_file = optarg;
+		break;
+	case 'j':
+		g_json_file = optarg;
+		break;
+	case 'N':
+		g_valid_ns_only = true;
+		break;
+	case 'S':
+		error_test = spdk_strtol(arg, 10);
+		if (error_test < 0) {
+			fprintf(stderr, "Invalid value supplied for the random seed.\n");
+			return -1;
+		} else {
+			g_seed_value = error_test;
+		}
+		break;
+	case 't':
+		g_runtime = spdk_strtol(optarg, 10);
+		if (g_runtime < 0 || g_runtime > MAX_RUNTIME_S) {
+			fprintf(stderr, "You must supply a positive runtime value less than 86401.\n");
+			return -1;
+		}
+		break;
+	case 'v':
+		g_verbose_mode = true;
+		break;
+	case '?':
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int
+main(int argc, char **argv)
+{
+	struct spdk_app_opts opts = {};
+	int rc;
+
+	spdk_app_opts_init(&opts);
+	opts.name = "nvme_fuzz";
+
+	g_runtime = DEFAULT_RUNTIME;
+	g_run = true;
+
+	if ((rc = spdk_app_parse_args(argc, argv, &opts, "aC:j:NS:t:v", NULL, nvme_fuzz_parse,
+				      nvme_fuzz_usage) != SPDK_APP_PARSE_ARGS_SUCCESS)) {
+		return rc;
+	}
+
+	if (g_conf_file) {
+		parse_trids();
+	}
+
+	if (g_json_file != NULL) {
+		g_cmd_array_size = fuzz_parse_args_into_array(g_json_file, (void **)&g_cmd_array,
+				   sizeof(struct spdk_nvme_cmd), g_nvme_cmd_json_name, parse_nvme_cmd_obj);
+		if (g_cmd_array_size == 0) {
+			fprintf(stderr, "The provided json file did not contain any valid commands. Exiting.");
+			return -EINVAL;
+		}
+	}
+
+	rc = spdk_app_start(&opts, begin_fuzz, NULL);
+
+	return rc;
+}

test/nvmf/nvmf.sh

@@ -30,6 +30,7 @@ run_test suite test/nvmf/target/bdev_io_wait.sh $TEST_ARGS
 run_test suite test/nvmf/target/create_transport.sh $TEST_ARGS
 
 if [ $RUN_NIGHTLY -eq 1 ]; then
+	run_test suite test/nvmf/target/fuzz.sh $TEST_ARGS
 	run_test suite test/nvmf/target/multiconnection.sh $TEST_ARGS
 fi
 

test/nvmf/target/fuzz.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+source $rootdir/test/nvmf/common.sh
+
+rpc_py="$rootdir/scripts/rpc.py"
+
+nvmftestinit
+
+timing_enter fuzz_test
+
+$NVMF_APP -m 0xF >$output_dir/nvmf_fuzz_tgt_output.txt 2>&1 &
+nvmfpid=$!
+
+trap "process_shm --id $NVMF_APP_SHM_ID; rm -f $testdir/nvmf_fuzz.conf; killprocess $nvmfpid; nvmftestfini $1; exit 1" SIGINT SIGTERM EXIT
+
+waitforlisten $nvmfpid
+$rpc_py nvmf_create_transport -t $TEST_TRANSPORT -u 8192
+
+$rpc_py construct_malloc_bdev -b Malloc0 64 512
+
+$rpc_py nvmf_subsystem_create nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001
+$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0
+$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+
+echo "[Nvme]" > $testdir/nvmf_fuzz.conf
+echo "  TransportID \"trtype:$TEST_TRANSPORT adrfam:IPv4 subnqn:nqn.2016-06.io.spdk:cnode1 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT\" Nvme0" >> $testdir/nvmf_fuzz.conf
+
+# Note that we chose a consistent seed to ensure that this test is consistent in nightly builds.
+$rootdir/test/app/fuzz/nvme_fuzz/nvme_fuzz -m 0xF0 -r "/var/tmp/nvme_fuzz" -t 30 -S 123456 -C $testdir/nvmf_fuzz.conf -N -a 2>$output_dir/nvmf_fuzz_logs1.txt
+# We don't specify a seed for this test. Instead we run a static list of commands from example.json.
+$rootdir/test/app/fuzz/nvme_fuzz/nvme_fuzz -m 0xF0 -r "/var/tmp/nvme_fuzz" -C $testdir/nvmf_fuzz.conf -j $rootdir/test/app/fuzz/nvme_fuzz/example.json -a 2>$output_dir/nvmf_fuzz_logs2.txt
+
+rm -f $testdir/nvmf_fuzz.conf
+$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1
+
+trap - SIGINT SIGTERM EXIT
+
+nvmfcleanup
+killprocess $nvmfpid
+
+nvmftestfini
+timing_exit fuzz_test