autotest: introduce SPDK_RUN_FUNCTIONAL_TEST

Introduced a new variable to run functional tests.
It's enabled by default, and can be manually disabled
on systems where e.g. only unit tests are run.

SPDK_RUN_FUNCTIONAL_TEST is a supplement to SPDK_UNITTEST.
The two are completely independent - both can be enabled,
disabled, or run in any combination.

The new variable is prefixed SPDK_RUN_ as it aligns nicely
with SPDK_RUN_CHECK_FORMAT, SPDK_RUN_VALGRIND, and
SPDK_RUN_ASAN, all of which control how much is tested.
SPDK_UNITTEST should eventually follow the same pattern
as well.

This gives us 2 layers of configuration:
SPDK_TEST_* <- what is tested
SPDK_RUN_* <- how it is tested

The following would run UT+ASAN for FTL and BlobFS, without
running their functional tests:

```
SPDK_RUN_FUNCTIONAL_TEST=0
SPDK_RUN_ASAN=1
SPDK_TEST_UNITTEST=1
SPDK_TEST_FTL=1
SPDK_TEST_BLOBFS=1
```

Change-Id: I9e592fa41aa2df8e246eca2bb9161b6da6832130
Signed-off-by: Seth Howell <seth.howell@intel.com>
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/c/442327
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Seth Howell <seth.howell5141@gmail.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

autotest.sh

@@ -105,104 +105,106 @@ if [ $SPDK_TEST_UNITTEST -eq 1 ]; then
 	timing_exit unittest
 fi
 
-timing_enter lib
 
-if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
-	run_test suite test/bdev/blockdev.sh
-fi
+if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
+	timing_enter lib
 
-if [ $SPDK_TEST_JSON -eq 1 ]; then
-	run_test suite test/config_converter/test_converter.sh
-fi
+	run_test suite test/env/env.sh
+	run_test suite test/rpc_client/rpc_client.sh
+	run_test suite ./test/json_config/json_config.sh
 
-if [ $SPDK_TEST_EVENT -eq 1 ]; then
-	run_test suite test/event/event.sh
-fi
-
-if [ $SPDK_TEST_NVME -eq 1 ]; then
-	run_test suite test/nvme/nvme.sh
-	if [ $SPDK_TEST_NVME_CLI -eq 1 ]; then
-		run_test suite test/nvme/spdk_nvme_cli.sh
+	if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
+		run_test suite test/bdev/blockdev.sh
+	fi
+
+	if [ $SPDK_TEST_JSON -eq 1 ]; then
+		run_test suite test/config_converter/test_converter.sh
+	fi
+
+	if [ $SPDK_TEST_EVENT -eq 1 ]; then
+		run_test suite test/event/event.sh
+	fi
+
+	if [ $SPDK_TEST_NVME -eq 1 ]; then
+		run_test suite test/nvme/nvme.sh
+		if [ $SPDK_TEST_NVME_CLI -eq 1 ]; then
+			run_test suite test/nvme/spdk_nvme_cli.sh
+		fi
+		# Only test hotplug without ASAN enabled. Since if it is
+		# enabled, it catches SEGV earlier than our handler which
+		# breaks the hotplug logic.
+		# Temporary workaround for issue #542, annotated for no VM image.
+		#if [ $SPDK_RUN_ASAN -eq 0 ]; then
+		#	run_test suite test/nvme/hotplug.sh intel
+		#fi
+	fi
+
+	if [ $SPDK_TEST_IOAT -eq 1 ]; then
+		run_test suite test/ioat/ioat.sh
+	fi
+
+	timing_exit lib
+
+	if [ $SPDK_TEST_ISCSI -eq 1 ]; then
+		run_test suite ./test/iscsi_tgt/iscsi_tgt.sh posix
+		run_test suite ./test/spdkcli/iscsi.sh
+	fi
+
+	if [ $SPDK_TEST_BLOBFS -eq 1 ]; then
+		run_test suite ./test/blobfs/rocksdb/rocksdb.sh
+		run_test suite ./test/blobstore/blobstore.sh
+	fi
+
+	if [ $SPDK_TEST_NVMF -eq 1 ]; then
+		run_test suite ./test/nvmf/nvmf.sh
+		run_test suite ./test/spdkcli/nvmf.sh
+	fi
+
+	if [ $SPDK_TEST_VHOST -eq 1 ]; then
+		run_test suite ./test/vhost/vhost.sh
+		report_test_completion "vhost"
+	fi
+
+	if [ $SPDK_TEST_LVOL -eq 1 ]; then
+		timing_enter lvol
+		test_cases="1,50,51,52,53,100,101,102,150,200,201,250,251,252,253,254,255,"
+		test_cases+="300,301,450,451,452,550,551,552,553,"
+		test_cases+="600,601,650,651,652,654,655,"
+		test_cases+="700,701,702,750,751,752,753,754,755,756,757,758,759,760,"
+		test_cases+="800,801,802,803,804,10000"
+		run_test suite ./test/lvol/lvol.sh --test-cases=$test_cases
+		run_test suite ./test/blobstore/blob_io_wait/blob_io_wait.sh
+		report_test_completion "lvol"
+		timing_exit lvol
+	fi
+
+	if [ $SPDK_TEST_VHOST_INIT -eq 1 ]; then
+		timing_enter vhost_initiator
+		run_test suite ./test/vhost/initiator/blockdev.sh
+		run_test suite ./test/spdkcli/virtio.sh
+		run_test suite ./test/vhost/shared/shared.sh
+		report_test_completion "vhost_initiator"
+		timing_exit vhost_initiator
+	fi
+
+	if [ $SPDK_TEST_PMDK -eq 1 ]; then
+		run_test suite ./test/pmem/pmem.sh -x
+		run_test suite ./test/spdkcli/pmem.sh
+	fi
+
+	if [ $SPDK_TEST_RBD -eq 1 ]; then
+		run_test suite ./test/spdkcli/rbd.sh
+	fi
+
+	if [ $SPDK_TEST_OCF -eq 1 ]; then
+		run_test suite ./test/ocf/ocf.sh
+	fi
+
+	if [ $SPDK_TEST_BDEV_FTL -eq 1 ]; then
+		run_test suite ./test/ftl/ftl.sh
 	fi
-	# Only test hotplug without ASAN enabled. Since if it is
-	# enabled, it catches SEGV earlier than our handler which
-	# breaks the hotplug logic.
-	# Temporary workaround for issue #542, annotated for no VM image.
-	#if [ $SPDK_RUN_ASAN -eq 0 ]; then
-	#	run_test suite test/nvme/hotplug.sh intel
-	#fi
 fi
 
-run_test suite test/env/env.sh
-run_test suite test/rpc_client/rpc_client.sh
-
-if [ $SPDK_TEST_IOAT -eq 1 ]; then
-	run_test suite test/ioat/ioat.sh
-fi
-
-timing_exit lib
-
-if [ $SPDK_TEST_ISCSI -eq 1 ]; then
-	run_test suite ./test/iscsi_tgt/iscsi_tgt.sh posix
-	run_test suite ./test/spdkcli/iscsi.sh
-fi
-
-if [ $SPDK_TEST_BLOBFS -eq 1 ]; then
-	run_test suite ./test/blobfs/rocksdb/rocksdb.sh
-	run_test suite ./test/blobstore/blobstore.sh
-fi
-
-if [ $SPDK_TEST_NVMF -eq 1 ]; then
-	run_test suite ./test/nvmf/nvmf.sh
-	run_test suite ./test/spdkcli/nvmf.sh
-fi
-
-if [ $SPDK_TEST_VHOST -eq 1 ]; then
-	run_test suite ./test/vhost/vhost.sh
-	report_test_completion "vhost"
-fi
-
-if [ $SPDK_TEST_LVOL -eq 1 ]; then
-	timing_enter lvol
-	test_cases="1,50,51,52,53,100,101,102,150,200,201,250,251,252,253,254,255,"
-	test_cases+="300,301,450,451,452,550,551,552,553,"
-	test_cases+="600,601,650,651,652,654,655,"
-	test_cases+="700,701,702,750,751,752,753,754,755,756,757,758,759,760,"
-	test_cases+="800,801,802,803,804,10000"
-	run_test suite ./test/lvol/lvol.sh --test-cases=$test_cases
-	run_test suite ./test/blobstore/blob_io_wait/blob_io_wait.sh
-	report_test_completion "lvol"
-	timing_exit lvol
-fi
-
-if [ $SPDK_TEST_VHOST_INIT -eq 1 ]; then
-	timing_enter vhost_initiator
-	run_test suite ./test/vhost/initiator/blockdev.sh
-	run_test suite ./test/spdkcli/virtio.sh
-	run_test suite ./test/vhost/shared/shared.sh
-	report_test_completion "vhost_initiator"
-	timing_exit vhost_initiator
-fi
-
-if [ $SPDK_TEST_PMDK -eq 1 ]; then
-	run_test suite ./test/pmem/pmem.sh -x
-	run_test suite ./test/spdkcli/pmem.sh
-fi
-
-if [ $SPDK_TEST_RBD -eq 1 ]; then
-	run_test suite ./test/spdkcli/rbd.sh
-fi
-
-if [ $SPDK_TEST_OCF -eq 1 ]; then
-	run_test suite ./test/ocf/ocf.sh
-fi
-
-if [ $SPDK_TEST_BDEV_FTL -eq 1 ]; then
-	run_test suite ./test/ftl/ftl.sh
-fi
-
-run_test suite ./test/json_config/json_config.sh
-
 timing_enter cleanup
 autotest_cleanup
 timing_exit cleanup

scripts/vagrant/autorun-spdk.conf

@@ -3,6 +3,7 @@ SPDK_BUILD_DOC=1
 SPDK_RUN_CHECK_FORMAT=1
 SPDK_RUN_SCANBUILD=1
 SPDK_RUN_VALGRIND=1
+SPDK_RUN_FUNCTIONAL_TEST=1
 SPDK_TEST_UNITTEST=1
 SPDK_TEST_ISAL=1
 SPDK_TEST_ISCSI=0

test/common/autotest_common.sh

@@ -39,6 +39,7 @@ fi
 : ${SPDK_RUN_CHECK_FORMAT=1}; export SPDK_RUN_CHECK_FORMAT
 : ${SPDK_RUN_SCANBUILD=1}; export SPDK_RUN_SCANBUILD
 : ${SPDK_RUN_VALGRIND=1}; export SPDK_RUN_VALGRIND
+: ${SPDK_RUN_FUNCTIONAL_TEST=1}; export SPDK_RUN_FUNCTIONAL_TEST
 : ${SPDK_TEST_UNITTEST=1}; export SPDK_TEST_UNITTEST
 : ${SPDK_TEST_ISAL=1}; export SPDK_TEST_ISAL
 : ${SPDK_TEST_ISCSI=1}; export SPDK_TEST_ISCSI

test/common/config/vm_setup.sh

@@ -459,6 +459,7 @@ SPDK_RUN_CHECK_FORMAT=1
 SPDK_RUN_SCANBUILD=1
 SPDK_RUN_VALGRIND=1
 SPDK_TEST_CRYPTO=1
+SPDK_RUN_FUNCTIONAL_TEST=1
 SPDK_TEST_UNITTEST=1
 SPDK_TEST_ISCSI=1
 SPDK_TEST_ISCSI_INITIATOR=1