From d5dc6a885387ad4d36c2fb11935f4ee0c7a28f72 Mon Sep 17 00:00:00 2001 From: Sebastian Brzezinka Date: Wed, 17 Aug 2022 08:05:00 +0200 Subject: [PATCH] test/llvm_vfio_fuzz: test normal IO during fuzzing Second vfio-user ctrlr is use to perform normal IO while fuzzing other one, misbehaving ctrlr should not affect IO operation. -Y Path of addition controller to perform normal io Signed-off-by: Sebastian Brzezinka Change-Id: I11c34e97723f9359bacd7866a9828a6d89c74992 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13882 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz.c | 227 +++++++++++++++++- test/nvmf/target/fuzz_vfio_json.conf | 45 ++++ test/nvmf/target/llvm_vfio_fuzz.sh | 8 +- 3 files changed, 266 insertions(+), 14 deletions(-) diff --git a/test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz.c b/test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz.c index 7c1798469..79286398e 100644 --- a/test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz.c +++ b/test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz.c @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) Intel Corporation. All rights reserved. */ - #include "spdk/stdinc.h" #include "spdk/conf.h" #include "spdk/env.h" @@ -36,9 +35,24 @@ static uint8_t *g_repro_data; static size_t g_repro_size; static pthread_t g_fuzz_td; static pthread_t g_reactor_td; -static bool g_in_fuzzer; static struct fuzz_type *g_fuzzer; +struct io_thread { + int lba_num; + char *write_buf; + char *read_buf; + size_t buf_size; + struct spdk_poller *run_poller; + struct spdk_thread *thread; + struct spdk_nvme_ctrlr *io_ctrlr; + pthread_t io_td; + struct spdk_nvme_ns *io_ns; + struct spdk_nvme_qpair *io_qpair; + char *io_ctrlr_path; + bool io_processing; + bool terminate; +} g_io_thread; + static int fuzz_vfio_user_version(const uint8_t *data, size_t size, struct vfio_device *dev) { @@ -111,13 +125,23 @@ TestOneInput(const uint8_t *data, size_t size) int LLVMFuzzerRunDriver(int *argc, char ***argv, int (*UserCb)(const uint8_t *Data, size_t Size)); +static void +io_terminate(void *ctx) +{ + ((struct io_thread *)ctx)->terminate = true; +} + static void exit_handler(void) { - if (g_in_fuzzer) { + if (g_io_thread.io_ctrlr_path) { + spdk_thread_send_msg(g_io_thread.thread, io_terminate, &g_io_thread); + + } else { spdk_app_stop(0); - pthread_join(g_reactor_td, NULL); } + + pthread_join(g_reactor_td, NULL); } static void * @@ -145,7 +169,6 @@ start_fuzzer(void *ctx) argv[argc - 2] = time_str; argv[argc - 1] = g_corpus_dir; - g_in_fuzzer = true; atexit(exit_handler); if (g_repro_data) { @@ -156,14 +179,181 @@ start_fuzzer(void *ctx) LLVMFuzzerRunDriver(&argc, &argv, TestOneInput); /* TODO: in the normal case, LLVMFuzzerRunDriver never returns - it calls exit() * directly and we never get here. But this behavior isn't really documented - * anywhere by LLVM, so call spdk_app_stop(0) if it does return, which will - * result in the app exiting like a normal SPDK application (spdk_app_start() - * returns to main(). + * anywhere by LLVM. */ } - g_in_fuzzer = false; - spdk_app_stop(0); + return NULL; +} + +static void +read_complete(void *arg, const struct spdk_nvme_cpl *completion) +{ + int sectors_num = 0; + struct io_thread *io = (struct io_thread *)arg; + + if (spdk_nvme_cpl_is_error(completion)) { + spdk_nvme_qpair_print_completion(io->io_qpair, (struct spdk_nvme_cpl *)completion); + fprintf(stderr, "I/O read error status: %s\n", + spdk_nvme_cpl_get_status_string(&completion->status)); + io->io_processing = false; + pthread_kill(g_fuzz_td, SIGSEGV); + return; + } + + if (memcmp(io->read_buf, io->write_buf, io->buf_size)) { + fprintf(stderr, "I/O corrupt, value not the same\n"); + pthread_kill(g_fuzz_td, SIGSEGV); + } + + sectors_num = spdk_nvme_ns_get_num_sectors(io->io_ns); + io->lba_num = (io->lba_num + 1) % sectors_num; + + io->io_processing = false; +} + +static void +write_complete(void *arg, const struct spdk_nvme_cpl *completion) +{ + int rc = 0; + struct io_thread *io = (struct io_thread *)arg; + + if (spdk_nvme_cpl_is_error(completion)) { + spdk_nvme_qpair_print_completion(io->io_qpair, + (struct spdk_nvme_cpl *)completion); + fprintf(stderr, "I/O write error status: %s\n", + spdk_nvme_cpl_get_status_string(&completion->status)); + io->io_processing = false; + pthread_kill(g_fuzz_td, SIGSEGV); + return; + } + rc = spdk_nvme_ns_cmd_read(io->io_ns, io->io_qpair, + io->read_buf, io->lba_num, 1, + read_complete, io, 0); + if (rc != 0) { + fprintf(stderr, "starting read I/O failed\n"); + io->io_processing = false; + pthread_kill(g_fuzz_td, SIGSEGV); + } +} + +static int +io_poller(void *ctx) +{ + int ret = 0; + struct io_thread *io = (struct io_thread *)ctx; + size_t i; + unsigned int seed = 0; + int *write_buf = (int *)io->write_buf; + + if (io->io_processing) { + spdk_nvme_qpair_process_completions(io->io_qpair, 0); + return SPDK_POLLER_BUSY; + } + + if (io->terminate) { + /* detaching controller here cause deadlock */ + spdk_poller_unregister(&(io->run_poller)); + spdk_free(io->write_buf); + spdk_free(io->read_buf); + + spdk_app_stop(0); + + return SPDK_POLLER_IDLE; + } + + /* Compiler should optimize the "/ sizeof(int)" into a right shift. */ + for (i = 0; i < io->buf_size / sizeof(int); i++) { + write_buf[i] = rand_r(&seed); + } + + io->io_processing = true; + + ret = spdk_nvme_ns_cmd_write(io->io_ns, io->io_qpair, + io->write_buf, io->lba_num, 1, + write_complete, io, 0); + if (ret < 0) { + fprintf(stderr, "starting write I/O failed\n"); + pthread_kill(g_fuzz_td, SIGSEGV); + return SPDK_POLLER_IDLE; + } + + return SPDK_POLLER_IDLE; +} + +static void +start_io_poller(void *ctx) +{ + struct io_thread *io = (struct io_thread *)ctx; + + io->run_poller = SPDK_POLLER_REGISTER(io_poller, ctx, 0); + if (io->run_poller == NULL) { + fprintf(stderr, "Failed to register a poller for IO.\n"); + spdk_app_stop(-1); + pthread_kill(g_fuzz_td, SIGSEGV); + } +} + +static void * +init_io(void *ctx) +{ + struct spdk_nvme_transport_id trid = {}; + int nsid = 0; + + snprintf(trid.traddr, sizeof(trid.traddr), "%s", g_io_thread.io_ctrlr_path); + + trid.trtype = SPDK_NVME_TRANSPORT_VFIOUSER; + g_io_thread.io_ctrlr = spdk_nvme_connect(&trid, NULL, 0); + if (g_io_thread.io_ctrlr == NULL) { + fprintf(stderr, "spdk_nvme_connect() failed for transport address '%s'\n", + trid.traddr); + spdk_app_stop(-1); + pthread_kill(g_fuzz_td, SIGSEGV); + return NULL; + } + + g_io_thread.io_qpair = spdk_nvme_ctrlr_alloc_io_qpair(g_io_thread.io_ctrlr, NULL, 0); + if (g_io_thread.io_qpair == NULL) { + spdk_nvme_detach(g_io_thread.io_ctrlr); + fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair failed\n"); + spdk_app_stop(-1); + pthread_kill(g_fuzz_td, SIGSEGV); + return NULL; + } + + if (spdk_nvme_ctrlr_get_num_ns(g_io_thread.io_ctrlr) == 0) { + fprintf(stderr, "no namespaces for IO\n"); + spdk_app_stop(-1); + pthread_kill(g_fuzz_td, SIGSEGV); + return NULL; + } + + nsid = spdk_nvme_ctrlr_get_first_active_ns(g_io_thread.io_ctrlr); + g_io_thread.io_ns = spdk_nvme_ctrlr_get_ns(g_io_thread.io_ctrlr, nsid); + if (!g_io_thread.io_ns) { + fprintf(stderr, "no io_ns for IO\n"); + spdk_app_stop(-1); + pthread_kill(g_fuzz_td, SIGSEGV); + return NULL; + } + + g_io_thread.buf_size = spdk_nvme_ns_get_sector_size(g_io_thread.io_ns); + + g_io_thread.read_buf = spdk_zmalloc(g_io_thread.buf_size, 0x1000, NULL, + SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); + + g_io_thread.write_buf = spdk_zmalloc(g_io_thread.buf_size, 0x1000, NULL, + SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); + + if (!g_io_thread.write_buf || !g_io_thread.read_buf) { + fprintf(stderr, "cannot allocated memory for io buffers\n"); + spdk_app_stop(-1); + pthread_kill(g_fuzz_td, SIGSEGV); + return NULL; + } + + g_io_thread.thread = spdk_thread_create("io_thread", NULL); + spdk_thread_send_msg(g_io_thread.thread, start_io_poller, &g_io_thread); return NULL; } @@ -174,6 +364,13 @@ begin_fuzz(void *ctx) g_reactor_td = pthread_self(); pthread_create(&g_fuzz_td, NULL, start_fuzzer, NULL); + + /* posix thread is use to avoid deadlock during spdk_nvme_connect + * vfio-user version negotiation may block when waiting for response + */ + if (g_io_thread.io_ctrlr_path) { + pthread_create(&g_io_thread.io_td, NULL, init_io, NULL); + } } static void @@ -183,6 +380,7 @@ vfio_fuzz_usage(void) fprintf(stderr, " -F Path for ctrlr that should be fuzzed.\n"); fprintf(stderr, " -N Name of reproduction data file.\n"); fprintf(stderr, " -t Time to run fuzz tests (in seconds). Default: 10\n"); + fprintf(stderr, " -Y Path of addition controller to perform io.\n"); fprintf(stderr, " -Z Fuzzer to run (0 to %lu)\n", NUM_FUZZERS - 1); } @@ -219,6 +417,13 @@ vfio_fuzz_parse(int ch, char *arg) return -1; } break; + case 'Y': + g_io_thread.io_ctrlr_path = strdup(optarg); + if (!g_io_thread.io_ctrlr_path) { + fprintf(stderr, "cannot strdup: %s\n", optarg); + return -ENOMEM; + } + break; case 't': case 'Z': tmp = spdk_strtoll(optarg, 10); @@ -272,7 +477,7 @@ main(int argc, char **argv) opts.name = "vfio_fuzz"; opts.shutdown_cb = fuzz_shutdown; - if ((rc = spdk_app_parse_args(argc, argv, &opts, "D:F:N:t:Z:", NULL, vfio_fuzz_parse, + if ((rc = spdk_app_parse_args(argc, argv, &opts, "D:F:N:t:Y:Z:", NULL, vfio_fuzz_parse, vfio_fuzz_usage) != SPDK_APP_PARSE_ARGS_SUCCESS)) { return rc; } diff --git a/test/nvmf/target/fuzz_vfio_json.conf b/test/nvmf/target/fuzz_vfio_json.conf index 36123a3b5..c899bcf12 100644 --- a/test/nvmf/target/fuzz_vfio_json.conf +++ b/test/nvmf/target/fuzz_vfio_json.conf @@ -12,6 +12,15 @@ "uuid": "6d6a0bf0-b712-40a7-8730-8f45797cc355" } }, + { + "method": "bdev_malloc_create", + "params": { + "name": "Malloc1", + "num_blocks": 131072, + "block_size": 512, + "uuid": "6d6a0bf0-b712-40a7-8730-8f45797cc356" + } + }, { "method": "bdev_wait_for_examine" } @@ -60,6 +69,19 @@ "ana_reporting": false } }, + { + "method": "nvmf_create_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode2", + "allow_any_host": true, + "serial_number": "SPDK00000000000001", + "model_number": "SPDK bdev Controller", + "max_namespaces": 32, + "min_cntlid": 1, + "max_cntlid": 65519, + "ana_reporting": false + } + }, { "method": "nvmf_subsystem_add_ns", "params": { @@ -72,6 +94,18 @@ } } }, + { + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode2", + "namespace": { + "nsid": 1, + "bdev_name": "Malloc1", + "nguid": "6D6A0BF0B71240A787308F45797CC356", + "uuid": "6d6a0bf0-b712-40a7-8730-8f45797cc356" + } + } + }, { "method": "nvmf_subsystem_add_listener", "params": { @@ -82,6 +116,17 @@ "trsvcid": "0" } } + }, + { + "method": "nvmf_subsystem_add_listener", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode2", + "listen_address": { + "trtype": "VFIOUSER", + "traddr": "/tmp/vfio-user/domain/2", + "trsvcid": "0" + } + } } ] } diff --git a/test/nvmf/target/llvm_vfio_fuzz.sh b/test/nvmf/target/llvm_vfio_fuzz.sh index e23fee5d7..a16351635 100755 --- a/test/nvmf/target/llvm_vfio_fuzz.sh +++ b/test/nvmf/target/llvm_vfio_fuzz.sh @@ -14,14 +14,16 @@ for i in "$@"; do done VFIOUSER_DIR=/tmp/vfio-user/domain/1 +VFIOUSER_IO_DIR=/tmp/vfio-user/domain/2 mkdir -p $VFIOUSER_DIR +mkdir -p $VFIOUSER_IO_DIR function start_llvm_fuzz() { local fuzzer_type=$1 local corpus_dir corpus_dir=/tmp/llvm_fuzz$fuzzer_type mkdir -p $corpus_dir - $rootdir/test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz -m 0x1 -i 0 -F $VFIOUSER_DIR -c $testdir/fuzz_vfio_json.conf -t $TIME -D $corpus_dir -Z $fuzzer_type + $rootdir/test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz -m 0x1 -i 0 -F $VFIOUSER_DIR -c $testdir/fuzz_vfio_json.conf -t $TIME -D $corpus_dir -Y $VFIOUSER_IO_DIR -Z $fuzzer_type } function run_fuzz() { @@ -53,7 +55,7 @@ fuzzfile=$rootdir/test/app/fuzz/llvm_vfio_fuzz/llvm_vfio_fuzz.c fuzz_num=$(($(grep -c "fn =" $fuzzfile) - 1)) [[ $fuzz_num -ne 0 ]] -trap 'process_shm --id 0; rm -rf /tmp/llvm_fuzz* $VFIOUSER_DIR; exit 1' SIGINT SIGTERM EXIT +trap 'process_shm --id 0; rm -rf /tmp/llvm_fuzz* $VFIOUSER_DIR $VFIOUSER_IO_DIR; exit 1' SIGINT SIGTERM EXIT if [[ $SPDK_TEST_FUZZER_SHORT -eq 1 ]]; then for ((i = 0; i < fuzz_num; i++)); do @@ -65,5 +67,5 @@ else start_llvm_fuzz $1 fi -rm -rf /tmp/llvm_fuzz* $VFIOUSER_DIR +rm -rf /tmp/llvm_fuzz* $VFIOUSER_DIR $VFIOUSER_IO_DIR trap - SIGINT SIGTERM EXIT