test/nvme: add nvme_compliance test app

This is a rough first cut at an nvme_compliance test
application.  Initial focus will be on testing
the SPDK vfio-user target, but over time many of the
tests here will apply to TCP and RDMA as well (or
even running the test against a virtual nvme device
backed by vfio-user in a VM).

Usage:

test/nvme/compliance/nvme_compliance -r <trid>

There is also a test/nvme/compliance/compliance.sh
script that starts the vfio-user target with a single
malloc namespace, and runs the nvme_compliance
app against that target.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I2c19df07c9ad93a69420545ab825015f49957fc2

Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9599
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: John Levon <levon@movementarian.org>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>

test/nvme/Makefile

@@ -35,7 +35,8 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 DIRS-y = aer reset sgl e2edp overhead deallocated_value err_injection \
-	startup reserve simple_copy connect_stress boot_partition
+	startup reserve simple_copy connect_stress boot_partition \
+	compliance
 DIRS-$(CONFIG_NVME_CUSE) += cuse
 
 .PHONY: all clean $(DIRS-y)

test/nvme/compliance/.gitignore

@@ -0,0 +1 @@
+nvme_compliance

test/nvme/compliance/Makefile

@@ -0,0 +1,42 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+
+APP = nvme_compliance
+
+# spdk_cunit.h
+CFLAGS += -I$(SPDK_ROOT_DIR)/test
+LIBS += -lcunit
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk

test/nvme/compliance/compliance.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+source $rootdir/test/nvmf/common.sh
+
+MALLOC_BDEV_SIZE=64
+MALLOC_BLOCK_SIZE=512
+
+rpc_py="$rootdir/scripts/rpc.py"
+
+export TEST_TRANSPORT=VFIOUSER
+
+rm -rf /var/run/vfio-user
+
+# Start the target
+"${NVMF_APP[@]}" -m 0x2 &
+nvmfpid=$!
+echo "Process pid: $nvmfpid"
+
+trap 'killprocess $nvmfpid; exit 1' SIGINT SIGTERM EXIT
+waitforlisten $nvmfpid
+
+sleep 1
+
+nqn=nqn.2021-09.io.spdk:cnode0
+traddr=/var/run/vfio-user
+
+$rpc_py nvmf_create_transport -t $TEST_TRANSPORT
+
+mkdir -p $traddr
+
+$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b malloc0
+$rpc_py nvmf_create_subsystem $nqn -a -s spdk
+$rpc_py nvmf_subsystem_add_ns $nqn malloc0
+$rpc_py nvmf_subsystem_add_listener $nqn -t $TEST_TRANSPORT -a $traddr -s 0
+
+$testdir/nvme_compliance -r "trtype:$TEST_TRANSPORT traddr:$traddr subnqn:$nqn"
+
+killprocess $nvmfpid
+
+rm -rf /var/run/vfio-user
+
+trap - SIGINT SIGTERM EXIT

test/nvme/compliance/nvme_compliance.c

@@ -0,0 +1,170 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk_cunit.h"
+#include "spdk/log.h"
+#include "spdk/nvme.h"
+
+static struct spdk_nvme_transport_id g_trid;
+static const char *g_trid_str;
+
+struct status {
+	bool			done;
+	struct spdk_nvme_cpl	cpl;
+};
+
+static void
+wait_for_admin_completion(struct status *s, struct spdk_nvme_ctrlr *ctrlr)
+{
+	/* Timeout if command does not complete within 1 second. */
+	uint64_t timeout = spdk_get_ticks() + spdk_get_ticks_hz();
+
+	while (!s->done && spdk_get_ticks() < timeout) {
+		spdk_nvme_ctrlr_process_admin_completions(ctrlr);
+	}
+
+	if (!s->done) {
+		CU_ASSERT(false && "completion timeout");
+	}
+}
+
+static void
+test_cb(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+	struct status *s = ctx;
+
+	s->done = true;
+	s->cpl = *cpl;
+}
+
+/* Test that target correctly handles requests to delete admin SQ/CQ (QID = 0).
+ * Associated with issue #2172.
+ */
+static void
+delete_admin_queue(void)
+{
+	struct spdk_nvme_ctrlr *ctrlr;
+	struct spdk_nvme_cmd cmd;
+	struct status s;
+	int rc;
+
+	SPDK_CU_ASSERT_FATAL(spdk_nvme_transport_id_parse(&g_trid, g_trid_str) == 0);
+	ctrlr = spdk_nvme_connect(&g_trid, NULL, 0);
+	SPDK_CU_ASSERT_FATAL(ctrlr);
+
+	/* Try deleting SQ for QID 0 (admin queue).  This is invalid. */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.opc = SPDK_NVME_OPC_DELETE_IO_SQ;
+	cmd.cdw10_bits.delete_io_q.qid = 0; /* admin queue */
+
+	s.done = false;
+	rc = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0, test_cb, &s);
+	CU_ASSERT(rc == 0);
+
+	wait_for_admin_completion(&s, ctrlr);
+
+	CU_ASSERT(s.cpl.status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC);
+	CU_ASSERT(s.cpl.status.sc == 1); /* Invalid Queue Identifier */
+
+	/* Try deleting CQ for QID 0 (admin queue).  This is invalid. */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.opc = SPDK_NVME_OPC_DELETE_IO_CQ;
+	cmd.cdw10_bits.delete_io_q.qid = 0; /* admin queue */
+
+	s.done = false;
+	rc = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0, test_cb, &s);
+	CU_ASSERT(rc == 0);
+
+	wait_for_admin_completion(&s, ctrlr);
+
+	CU_ASSERT(s.cpl.status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC);
+	CU_ASSERT(s.cpl.status.sc == 1); /* Invalid Queue Identifier */
+
+	spdk_nvme_detach(ctrlr);
+}
+
+static int
+parse_args(int argc, char **argv, struct spdk_env_opts *opts)
+{
+	char op;
+
+	while ((op = getopt(argc, argv, "r:")) != -1) {
+		switch (op) {
+		case 'r':
+			g_trid_str = optarg;
+			break;
+		default:
+			SPDK_ERRLOG("Unknown op '%c'\n", op);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	struct spdk_env_opts	opts;
+	CU_pSuite		suite = NULL;
+	unsigned int		num_failures;
+
+	CU_set_error_action(CUEA_ABORT);
+	CU_initialize_registry();
+
+	suite = CU_add_suite("nvme_compliance", NULL, NULL);
+
+	spdk_env_opts_init(&opts);
+	opts.name = "nvme_compliance";
+	if (parse_args(argc, argv, &opts)) {
+		fprintf(stderr, "could not parse_args\n");
+		return -1;
+	}
+
+	if (g_trid_str == NULL) {
+		fprintf(stderr, "-t <trid> not specified\n");
+		return -1;
+	}
+
+	if (spdk_env_init(&opts)) {
+		fprintf(stderr, "could not spdk_env_init\n");
+		return -1;
+	}
+
+	CU_ADD_TEST(suite, delete_admin_queue);
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+	return num_failures;
+}