From b0ad3322f0f178ef5c816d6f8456257288efe81b Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Mon, 12 Dec 2016 15:42:43 +0800 Subject: [PATCH] bdev/nvme: RPC call can now specify remote NVMe-oF devices Change-Id: Ief3cf71489f0725249d51851b8d3270cb4e034db Signed-off-by: Ziye Yang --- autotest.sh | 1 + lib/bdev/nvme/blockdev_nvme.c | 3 + lib/bdev/nvme/blockdev_nvme_rpc.c | 54 +++++++++++-- scripts/rpc.py | 22 +++++- test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh | 77 +++++++++++++++++++ test/iscsi_tgt/nvme_remote/iscsi.conf | 15 ++++ test/iscsi_tgt/rpc_config/rpc_config.py | 2 +- 7 files changed, 164 insertions(+), 10 deletions(-) create mode 100755 test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh create mode 100644 test/iscsi_tgt/nvme_remote/iscsi.conf diff --git a/autotest.sh b/autotest.sh index 5abf5b615..9a67987c9 100755 --- a/autotest.sh +++ b/autotest.sh @@ -135,6 +135,7 @@ if [ $(uname -s) = Linux ] && [ $SPDK_TEST_ISCSI -eq 1 ]; then fi run_test ./test/iscsi_tgt/ext4test/ext4test.sh run_test ./test/iscsi_tgt/rbd/rbd.sh + run_test ./test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh timing_exit iscsi_tgt run_test test/lib/iscsi/iscsi.sh diff --git a/lib/bdev/nvme/blockdev_nvme.c b/lib/bdev/nvme/blockdev_nvme.c index 06aad92bd..4870e69c5 100644 --- a/lib/bdev/nvme/blockdev_nvme.c +++ b/lib/bdev/nvme/blockdev_nvme.c @@ -543,6 +543,7 @@ probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, } if (!claim_device) { + SPDK_ERRLOG("Not claiming device at %s\n", trid->traddr); return false; } @@ -675,11 +676,13 @@ spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, probe_ctx.count = 1; probe_ctx.trids[0] = *trid; if (spdk_nvme_probe(trid, &probe_ctx, probe_cb, attach_cb, NULL)) { + SPDK_ERRLOG("Failed to probe for new devices\n"); return -1; } nvme_ctrlr = nvme_ctrlr_get(trid); if (!nvme_ctrlr) { + SPDK_ERRLOG("Failed to find new NVMe controller\n"); return -1; } diff --git a/lib/bdev/nvme/blockdev_nvme_rpc.c b/lib/bdev/nvme/blockdev_nvme_rpc.c index f8a71c0b1..593a9d3e3 100644 --- a/lib/bdev/nvme/blockdev_nvme_rpc.c +++ b/lib/bdev/nvme/blockdev_nvme_rpc.c @@ -34,23 +34,38 @@ #include #include "blockdev_nvme.h" + +#include "spdk/string.h" #include "spdk/rpc.h" #include "spdk/util.h" #include "spdk_internal/log.h" struct rpc_construct_nvme { - char *pci_address; + char *trtype; + char *adrfam; + char *traddr; + char *trsvcid; + char *subnqn; }; static void free_rpc_construct_nvme(struct rpc_construct_nvme *req) { - free(req->pci_address); + free(req->trtype); + free(req->adrfam); + free(req->traddr); + free(req->trsvcid); + free(req->subnqn); } static const struct spdk_json_object_decoder rpc_construct_nvme_decoders[] = { - {"pci_address", offsetof(struct rpc_construct_nvme, pci_address), spdk_json_decode_string}, + {"trtype", offsetof(struct rpc_construct_nvme, trtype), spdk_json_decode_string}, + {"traddr", offsetof(struct rpc_construct_nvme, traddr), spdk_json_decode_string}, + + {"adrfam", offsetof(struct rpc_construct_nvme, adrfam), spdk_json_decode_string, true}, + {"trsvcid", offsetof(struct rpc_construct_nvme, trsvcid), spdk_json_decode_string, true}, + {"subnqn", offsetof(struct rpc_construct_nvme, subnqn), spdk_json_decode_string, true}, }; #define NVME_MAX_BLOCKDEVS_PER_RPC 32 @@ -66,16 +81,43 @@ spdk_rpc_construct_nvme_bdev(struct spdk_jsonrpc_server_conn *conn, const char *names[NVME_MAX_BLOCKDEVS_PER_RPC]; size_t count = 0; size_t i; + int rc; if (spdk_json_decode_object(params, rpc_construct_nvme_decoders, SPDK_COUNTOF(rpc_construct_nvme_decoders), &req)) { - SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n"); + SPDK_ERRLOG("spdk_json_decode_object failed\n"); goto invalid; } - trid.trtype = SPDK_NVME_TRANSPORT_PCIE; - snprintf(trid.traddr, sizeof(trid.traddr), "%s", req.pci_address); + /* Parse trtype */ + rc = spdk_nvme_transport_id_parse_trtype(&trid.trtype, req.trtype); + if (rc < 0) { + SPDK_ERRLOG("Failed to parse trtype: %s\n", req.trtype); + goto invalid; + } + + /* Parse traddr */ + snprintf(trid.traddr, sizeof(trid.traddr), "%s", req.traddr); + + /* Parse adrfam */ + if (req.adrfam) { + rc = spdk_nvme_transport_id_parse_adrfam(&trid.adrfam, req.adrfam); + if (rc < 0) { + SPDK_ERRLOG("Failed to parse adrfam: %s\n", req.adrfam); + goto invalid; + } + } + + /* Parse trsvcid */ + if (req.trsvcid) { + snprintf(trid.trsvcid, sizeof(trid.trsvcid), "%s", req.trsvcid); + } + + /* Parse subnqn */ + if (req.subnqn) { + snprintf(trid.subnqn, sizeof(trid.subnqn), "%s", req.subnqn); + } count = NVME_MAX_BLOCKDEVS_PER_RPC; if (spdk_bdev_nvme_create(&trid, names, &count)) { diff --git a/scripts/rpc.py b/scripts/rpc.py index 4bb7b78a1..5bd681201 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -173,10 +173,26 @@ p.add_argument('fname', help='Path to device or file (ex: /dev/sda)') p.set_defaults(func=construct_aio_bdev) def construct_nvme_bdev(args): - params = {'pci_address': args.pci_address} - print_array(jsonrpc_call('construct_nvme_bdev', params)) + params = {'trtype': args.trtype, + 'traddr': args.traddr} + + if args.adrfam: + params['adrfam'] = args.adrfam + + if args.trsvcid: + params['trsvcid'] = args.trsvcid + + if args.subnqn: + params['subnqn'] = args.subnqn + + jsonrpc_call('construct_nvme_bdev', params) + p = subparsers.add_parser('construct_nvme_bdev', help='Add bdev with nvme backend') -p.add_argument('pci_address', help='PCI address domain:bus:device.function') +p.add_argument('-t', '--trtype', help='NVMe-oF target trtype: e.g., rdma, pcie', required=True) +p.add_argument('-a', '--traddr', help='NVMe-oF target address: e.g., an ip address or BDF', required=True) +p.add_argument('-f', '--adrfam', help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host') +p.add_argument('-s', '--trsvcid', help='NVMe-oF target trsvcid: e.g., a port number') +p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn') p.set_defaults(func=construct_nvme_bdev) def construct_rbd_bdev(args): diff --git a/test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh b/test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh new file mode 100755 index 000000000..032b14334 --- /dev/null +++ b/test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -e + +testdir=$(readlink -f $(dirname $0)) +rootdir=$testdir/../../.. +source $rootdir/scripts/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +if [ -z "$TARGET_IP" ]; then + echo "TARGET_IP not defined in environment" + exit 1 +fi + +if [ -z "$INITIATOR_IP" ]; then + echo "INITIATOR_IP not defined in environment" + exit 1 +fi + +if ! rdma_nic_available; then + echo "no NIC for nvmf test" + exit 0 +fi + +rpc_py="python $rootdir/scripts/rpc.py" +fio_py="python $rootdir/scripts/fio.py" + +ISCSI_PORT=3260 +NVMF_PORT=4420 + +timing_enter nvme_remote + +# Start the NVMf target +$rootdir/app/nvmf_tgt/nvmf_tgt -c $rootdir/test/nvmf/nvmf.conf -m 0x2 -p 1 -s 512 & +nvmfpid=$! +echo "NVMf target launched. pid: $nvmfpid" +trap "killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT +waitforlisten $nvmfpid 5260 +echo "NVMf target has started." +bdevs=$($rpc_py construct_malloc_bdev 64 512) +$rpc_py construct_nvmf_subsystem Virtual nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -s SPDK00000000000001 -n "$bdevs" +echo "NVMf subsystem created." + +# Start the iSCSI target +$rootdir/app/iscsi_tgt/iscsi_tgt -c $testdir/iscsi.conf -m 0x1 -p 0 -s 512 & +iscsipid=$! +echo "iSCSI target launched. pid: $iscsipid" +trap "process_core; killprocess $iscsipid; exit 1" SIGINT SIGTERM EXIT +# The configuration file for the iSCSI target told it to use port 5261 for RPC +waitforlisten $iscsipid 5261 +echo "iSCSI target has started." + +echo "Creating an iSCSI target node." +$rpc_py -p 5261 add_portal_group 1 $TARGET_IP:$ISCSI_PORT +$rpc_py -p 5261 add_initiator_group 1 ALL $INITIATOR_IP/32 +$rpc_py -p 5261 construct_nvme_bdev -t "rdma" -f "ipv4" -a 192.168.100.8 -s $NVMF_PORT -n nqn.2016-06.io.spdk:cnode1 +$rpc_py -p 5261 construct_target_node Target1 Target1_alias 'Nvme0n1:0' '1:1' 64 1 0 0 0 +sleep 1 + +echo "Logging in to iSCSI target." +iscsiadm -m discovery -t sendtargets -p $TARGET_IP:$ISCSI_PORT +iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT +trap "iscsicleanup; process_core; killprocess $pid; exit 1" SIGINT SIGTERM EXIT +sleep 1 + +echo "Running FIO" +$fio_py 4096 1 randrw 1 verify + +rm -f ./local-job0-0-verify.state +trap - SIGINT SIGTERM EXIT + +iscsicleanup +killprocess $iscsipid +$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 +killprocess $nvmfpid + +timing_exit nvme_remote diff --git a/test/iscsi_tgt/nvme_remote/iscsi.conf b/test/iscsi_tgt/nvme_remote/iscsi.conf new file mode 100644 index 000000000..b59d534a0 --- /dev/null +++ b/test/iscsi_tgt/nvme_remote/iscsi.conf @@ -0,0 +1,15 @@ +[Global] + LogFacility "local7" + +[iSCSI] + NodeBase "iqn.2016-06.io.spdk" + AuthFile /usr/local/etc/spdk/auth.conf + Timeout 30 + DiscoveryAuthMethod Auto + MaxSessions 16 + ImmediateData Yes + ErrorRecoveryLevel 0 + +[Rpc] + Enable Yes + Listen 127.0.0.1:5261 diff --git a/test/iscsi_tgt/rpc_config/rpc_config.py b/test/iscsi_tgt/rpc_config/rpc_config.py index d7549b390..0d6733d70 100755 --- a/test/iscsi_tgt/rpc_config/rpc_config.py +++ b/test/iscsi_tgt/rpc_config/rpc_config.py @@ -399,7 +399,7 @@ def verify_add_nvme_bdev_rpc_methods(rpc_py): output = check_output(["lspci", "-mm", "-nn"]) addrs = re.findall('^([0-9]{2}:[0-9]{2}.[0-9]) "Non-Volatile memory controller \[0108\]".*-p02', output, re.MULTILINE) for addr in addrs: - ctrlr_address = "0000:{}".format(addr) + ctrlr_address = "-t pcie -a 0000:{}".format(addr) rpc.construct_nvme_bdev(ctrlr_address) print "add nvme device passed first time" test_pass = 0