ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test/nvme, test/nvmf: add fused_ordering test

Browse Source 
This test reproduced issue #2428. It does the following:

1) submit some large writes (requiring R2T or RDMA READ)
2) submit fused compare (first fused)
3) variable number of qpair polls
4) submit fused write (second fused)

The RDMA and TCP transports were not accounting for the
fused bits, and would send a non-fused command down to
the target layer between the two fused commands.

By adding the variable amount of delay between the fused
command submissions, it creates a window where a large
write's payload is fetched from the host and submitted
to the target between submission of the two fused commands.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I1abd5a6bca08386279c09f8135387826149d30ef
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11963
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
This commit is contained in:
Jim Harris 2022-03-12 16:06:09 +00:00 committed by Tomasz Zawadzki
parent 183c348557
commit 74cd76db68
6 changed files with 296 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/nvme/Makefile
View File

@ -36,7 +36,7 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = aer reset sgl e2edp overhead deallocated_value err_injection \
startup reserve simple_copy connect_stress boot_partition \
compliance
compliance fused_ordering
DIRS-$(CONFIG_NVME_CUSE) += cuse
.PHONY: all clean $(DIRS-y)

1
test/nvme/fused_ordering/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
fused_ordering

38
test/nvme/fused_ordering/Makefile Normal file
View File

@ -0,0 +1,38 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Samsung Electronics Co., Ltd. nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
APP = fused_ordering
include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk

231
test/nvme/fused_ordering/fused_ordering.c Normal file
View File

@ -0,0 +1,231 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/stdinc.h"
#include "spdk/nvme.h"
#include "spdk/env.h"
#include "spdk/string.h"
#include "spdk/log.h"
static struct spdk_nvme_ctrlr *g_ctrlr;
static struct spdk_nvme_ns *g_ns;
static struct spdk_nvme_qpair *g_qpair;
static struct spdk_nvme_transport_id g_trid = {};
static uint32_t g_outstanding;
static void
io_complete(void *arg, const struct spdk_nvme_cpl *cpl)
{
if (spdk_nvme_cpl_is_error(cpl)) {
spdk_nvme_print_completion(spdk_nvme_qpair_get_id(g_qpair),
(struct spdk_nvme_cpl *)cpl);
exit(1);
}
g_outstanding--;
}
#define WRITE_BLOCKS 128
#define FUSED_BLOCKS 1
static void
fused_ordering(uint32_t poll_count)
{
void *cw_buf, *large_buf;
int rc;
int i;
g_qpair = spdk_nvme_ctrlr_alloc_io_qpair(g_ctrlr, NULL, 0);
if (g_qpair == NULL) {
printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
exit(1);
}
cw_buf = spdk_zmalloc(FUSED_BLOCKS * 4096, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
if (cw_buf == NULL) {
printf("ERROR: buffer allocation failed\n");
return;
}
large_buf = spdk_zmalloc(WRITE_BLOCKS * 4096, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY,
SPDK_MALLOC_DMA);
if (large_buf == NULL) {
printf("ERROR: buffer allocation failed\n");
return;
}
/* Issue a bunch of relatively large writes - big enough that the data will not fit
* in-capsule - followed by the compare command. Then poll the completion queue a number of
* times matching the poll_count variable. This adds a variable amount of delay between
* the compare and the subsequent fused write submission.
*
* GitHub issue #2428 showed a problem where once the non-in-capsule data had been fetched from
* the host, that request could get sent to the target layer between the two fused commands. This
* variable delay would eventually induce this condition before the fix.
*/
for (i = 0; i < 8; i++) {
rc = spdk_nvme_ns_cmd_write(g_ns, g_qpair, large_buf, 0, WRITE_BLOCKS, io_complete, NULL, 0);
if (rc != 0) {
fprintf(stderr, "starting write I/O failed\n");
exit(1);
}
g_outstanding++;
}
rc = spdk_nvme_ns_cmd_compare(g_ns, g_qpair, cw_buf, 0, FUSED_BLOCKS, io_complete, NULL,
SPDK_NVME_IO_FLAGS_FUSE_FIRST);
if (rc != 0) {
fprintf(stderr, "starting compare I/O failed\n");
exit(1);
}
g_outstanding++;
while (poll_count--) {
spdk_nvme_qpair_process_completions(g_qpair, 0);
}
rc = spdk_nvme_ns_cmd_write(g_ns, g_qpair, cw_buf, 0, FUSED_BLOCKS, io_complete, NULL,
SPDK_NVME_IO_FLAGS_FUSE_SECOND);
if (rc != 0) {
fprintf(stderr, "starting write I/O failed\n");
exit(1);
}
g_outstanding++;
while (g_outstanding) {
spdk_nvme_qpair_process_completions(g_qpair, 0);
}
spdk_nvme_ctrlr_free_io_qpair(g_qpair);
spdk_free(cw_buf);
spdk_free(large_buf);
}
static void
usage(const char *program_name)
{
printf("%s [options]", program_name);
printf("\t\n");
printf("options:\n");
printf("\t[-r remote NVMe over Fabrics target address]\n");
#ifdef DEBUG
printf("\t[-L enable debug logging]\n");
#else
printf("\t[-L enable debug logging (flag disabled, must reconfigure with --enable-debug)\n");
#endif
}
static int
parse_args(int argc, char **argv, struct spdk_env_opts *env_opts)
{
int op, rc;
while ((op = getopt(argc, argv, "r:L:")) != -1) {
switch (op) {
case 'r':
if (spdk_nvme_transport_id_parse(&g_trid, optarg) != 0) {
fprintf(stderr, "Error parsing transport address\n");
return 1;
}
break;
case 'L':
rc = spdk_log_set_flag(optarg);
if (rc < 0) {
fprintf(stderr, "unknown flag\n");
usage(argv[0]);
exit(EXIT_FAILURE);
}
#ifdef DEBUG
spdk_log_set_print_level(SPDK_LOG_DEBUG);
#endif
break;
default:
usage(argv[0]);
return 1;
}
}
return 0;
}
int main(int argc, char **argv)
{
int rc, i;
struct spdk_env_opts opts;
struct spdk_nvme_ctrlr_opts ctrlr_opts;
int nsid;
spdk_env_opts_init(&opts);
spdk_log_set_print_level(SPDK_LOG_NOTICE);
rc = parse_args(argc, argv, &opts);
if (rc != 0) {
return rc;
}
opts.name = "fused_ordering";
if (spdk_env_init(&opts) < 0) {
fprintf(stderr, "Unable to initialize SPDK env\n");
return 1;
}
spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctrlr_opts, sizeof(ctrlr_opts));
ctrlr_opts.keep_alive_timeout_ms = 60 * 1000;
g_ctrlr = spdk_nvme_connect(&g_trid, &ctrlr_opts, sizeof(ctrlr_opts));
if (g_ctrlr == NULL) {
fprintf(stderr, "spdk_nvme_connect() failed\n");
rc = 1;
goto exit;
}
printf("Attached to %s\n", g_trid.subnqn);
nsid = spdk_nvme_ctrlr_get_first_active_ns(g_ctrlr);
if (nsid == 0) {
perror("No active namespaces");
exit(1);
}
g_ns = spdk_nvme_ctrlr_get_ns(g_ctrlr, nsid);
printf(" Namespace ID: %d size: %juGB\n", spdk_nvme_ns_get_id(g_ns),
spdk_nvme_ns_get_size(g_ns) / 1000000000);
for (i = 0; i < 1024; i++) {
printf("fused_ordering(%d)\n", i);
fused_ordering(i);
}
exit:
spdk_nvme_detach(g_ctrlr);
spdk_env_fini();
return rc;
}

1
test/nvmf/nvmf.sh
View File

@ -25,6 +25,7 @@ if [[ $SPDK_TEST_URING -eq 0 ]]; then
run_test "nvmf_abort" test/nvmf/target/abort.sh "${TEST_ARGS[@]}"
run_test "nvmf_ns_hotplug_stress" test/nvmf/target/ns_hotplug_stress.sh "${TEST_ARGS[@]}"
run_test "nvmf_connect_stress" test/nvmf/target/connect_stress.sh "${TEST_ARGS[@]}"
run_test "nvmf_fused_ordering" test/nvmf/target/fused_ordering.sh "${TEST_ARGS[@]}"
run_test "nvmf_delete_subsystem" test/nvmf/target/delete_subsystem.sh "${TEST_ARGS[@]}"
run_test "nvmf_multicontroller" test/nvmf/host/multicontroller.sh "${TEST_ARGS[@]}"
run_test "nvmf_aer" test/nvmf/host/aer.sh "${TEST_ARGS[@]}"

24
test/nvmf/target/fused_ordering.sh Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
testdir=$(readlink -f $(dirname $0))
rootdir=$(readlink -f $testdir/../../..)
source $rootdir/test/common/autotest_common.sh
source $rootdir/test/nvmf/common.sh
rpc_py="$rootdir/scripts/rpc.py"
nvmftestinit
nvmfappstart -m 0x2
$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192
$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -m 10
$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
$rpc_py bdev_null_create NULL1 1000 512
$rpc_py bdev_wait_for_examine
$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 NULL1
$rootdir/test/nvme/fused_ordering/fused_ordering -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT subnqn:nqn.2016-06.io.spdk:cnode1"
trap - SIGINT SIGTERM EXIT
nvmftestfini