diff --git a/CHANGELOG.md b/CHANGELOG.md index cb49d31db..be2521b47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,12 @@ The HotplugEnable option in `[Nvme]` sections of the configuration file is now The NVMe library now includes a function spdk_nvme_ns_get_ctrlr which returns the NVMe Controller associated with a given namespace. +The NVMe library now allows the user to specify a host identifier when attaching +to a controller. The host identifier is used as part of the Reservations feature, +as well as in the NVMe-oF Connect command. The default host ID is also now a +randomly-generated UUID, and the default host NQN uses the host ID to generate +a UUID-based NQN. + ### NVMe-oF Target (nvmf_tgt) The NVMe-oF target no longer requires any in capsule data buffers to run, and diff --git a/examples/nvme/reserve/reserve.c b/examples/nvme/reserve/reserve.c index 47d8fdea6..4bb70746d 100644 --- a/examples/nvme/reserve/reserve.c +++ b/examples/nvme/reserve/reserve.c @@ -33,8 +33,10 @@ #include "spdk/stdinc.h" +#include "spdk/endian.h" #include "spdk/nvme.h" #include "spdk/env.h" +#include "spdk/log.h" #define MAX_DEVS 64 @@ -52,44 +54,22 @@ static int num_devs = 0; static int outstanding_commands; static int reserve_command_result; -static int set_feature_result; - -struct feature { - uint32_t result; - bool valid; -}; - -static struct feature features[256]; +static bool get_host_id_successful; #define HOST_ID 0xABABABABCDCDCDCD +#define EXT_HOST_ID ((uint8_t[]){0x0f, 0x97, 0xcd, 0x74, 0x8c, 0x80, 0x41, 0x42, \ + 0x99, 0x0f, 0x65, 0xc4, 0xf0, 0x39, 0x24, 0x20}) + #define CR_KEY 0xDEADBEAF5A5A5A5B static void get_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl) { - struct feature *feature = cb_arg; - int fid = feature - features; - if (spdk_nvme_cpl_is_error(cpl)) { - fprintf(stdout, "get_feature(0x%02X) failed\n", fid); + fprintf(stdout, "Get Features - Host Identifier failed\n"); + get_host_id_successful = false; } else { - feature->result = cpl->cdw0; - feature->valid = true; - } - outstanding_commands--; -} - -static void -set_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl) -{ - struct feature *feature = cb_arg; - int fid = feature - features; - - if (spdk_nvme_cpl_is_error(cpl)) { - fprintf(stdout, "set_feature(0x%02X) failed\n", fid); - set_feature_result = -1; - } else { - set_feature_result = 0; + get_host_id_successful = true; } outstanding_commands--; } @@ -98,81 +78,40 @@ static int get_host_identifier(struct spdk_nvme_ctrlr *ctrlr) { int ret; - uint64_t *host_id; - struct spdk_nvme_cmd cmd = {}; + uint8_t host_id[16]; + uint32_t host_id_size; + uint32_t cdw11; - cmd.opc = SPDK_NVME_OPC_GET_FEATURES; - cmd.cdw10 = SPDK_NVME_FEAT_HOST_IDENTIFIER; - - host_id = spdk_dma_zmalloc(sizeof(*host_id), 0x1000, NULL); - if (!host_id) { - fprintf(stderr, "Host_ID DMA Buffer Allocation Failed\n"); - return -1; + if (spdk_nvme_ctrlr_get_data(ctrlr)->ctratt.host_id_exhid_supported) { + host_id_size = 16; + cdw11 = 1; + printf("Using 128-bit extended host identifier\n"); + } else { + host_id_size = 8; + cdw11 = 0; + printf("Using 64-bit host identifier\n"); } outstanding_commands = 0; - ret = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, host_id, sizeof(*host_id), - get_feature_completion, &features[SPDK_NVME_FEAT_HOST_IDENTIFIER]); + ret = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_HOST_IDENTIFIER, cdw11, host_id, + host_id_size, + get_feature_completion, NULL); if (ret) { fprintf(stdout, "Get Feature: Failed\n"); - spdk_dma_free(host_id); return -1; } outstanding_commands++; + get_host_id_successful = false; while (outstanding_commands) { spdk_nvme_ctrlr_process_admin_completions(ctrlr); } - if (features[SPDK_NVME_FEAT_HOST_IDENTIFIER].valid) { - fprintf(stdout, "Get Feature: Host Identifier 0x%" PRIx64 "\n", *host_id); + if (get_host_id_successful) { + spdk_trace_dump(stdout, "Get Feature: Host Identifier:", host_id, host_id_size); } - spdk_dma_free(host_id); - return 0; -} - -static int -set_host_identifier(struct spdk_nvme_ctrlr *ctrlr) -{ - int ret; - uint64_t *host_id; - struct spdk_nvme_cmd cmd = {}; - - cmd.opc = SPDK_NVME_OPC_SET_FEATURES; - cmd.cdw10 = SPDK_NVME_FEAT_HOST_IDENTIFIER; - - host_id = spdk_dma_zmalloc(sizeof(*host_id), 0x1000, NULL); - if (!host_id) { - fprintf(stderr, "Host_ID DMA Buffer Allocation Failed\n"); - return -1; - } - - *host_id = HOST_ID; - - outstanding_commands = 0; - set_feature_result = -1; - - fprintf(stdout, "Set Feature: Host Identifier 0x%" PRIx64 "\n", *host_id); - ret = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, host_id, sizeof(*host_id), - set_feature_completion, &features[SPDK_NVME_FEAT_HOST_IDENTIFIER]); - if (ret) { - fprintf(stdout, "Set Feature: Failed\n"); - spdk_dma_free(host_id); - return -1; - } - - outstanding_commands++; - - while (outstanding_commands) { - spdk_nvme_ctrlr_process_admin_completions(ctrlr); - } - - if (set_feature_result) - fprintf(stdout, "Set Feature: Host Identifier Failed\n"); - - spdk_dma_free(host_id); return 0; } @@ -369,7 +308,6 @@ reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, if (!cdata->oncs.reservations) return; - set_host_identifier(ctrlr); get_host_identifier(ctrlr); /* tested 1 namespace */ @@ -383,6 +321,15 @@ static bool probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, struct spdk_nvme_ctrlr_opts *opts) { + /* + * Provide both 64-bit and 128-bit host identifiers. + * + * The NVMe library will choose which one to use based on whether the controller + * supports extended host identifiers. + */ + to_le64(opts->host_id, HOST_ID); + memcpy(opts->extended_host_id, EXT_HOST_ID, sizeof(opts->extended_host_id)); + return true; } diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index 6fb8ec8ff..233f7dafc 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -130,6 +130,20 @@ struct spdk_nvme_ctrlr_opts { * specified. */ char src_svcid[SPDK_NVMF_TRSVCID_MAX_LEN + 1]; + + /** + * The host identifier to use when connecting to controllers with 64-bit host ID support. + * + * Set to all zeroes to specify that no host ID should be provided to the controller. + */ + uint8_t host_id[8]; + + /** + * The host identifier to use when connecting to controllers with extended (128-bit) host ID support. + * + * Set to all zeroes to specify that no host ID should be provided to the controller. + */ + uint8_t extended_host_id[16]; }; /** diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c index d09999fb9..b6500c926 100644 --- a/lib/nvme/nvme.c +++ b/lib/nvme/nvme.c @@ -34,6 +34,8 @@ #include "spdk/nvmf_spec.h" #include "nvme_internal.h" +#include + #define SPDK_NVME_DRIVER_NAME "spdk_nvme_driver" struct nvme_driver *g_spdk_nvme_driver; @@ -227,6 +229,7 @@ nvme_robust_mutex_init_shared(pthread_mutex_t *mtx) static int nvme_driver_init(void) { + uuid_t host_id; int ret = 0; /* Any socket ID */ int socket_id = -1; @@ -300,6 +303,11 @@ nvme_driver_init(void) TAILQ_INIT(&g_spdk_nvme_driver->init_ctrlrs); TAILQ_INIT(&g_spdk_nvme_driver->attached_ctrlrs); + SPDK_STATIC_ASSERT(sizeof(host_id) == sizeof(g_spdk_nvme_driver->default_extended_host_id), + "host ID size mismatch"); + uuid_generate(host_id); + memcpy(g_spdk_nvme_driver->default_extended_host_id, host_id, sizeof(host_id)); + nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); return ret; diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index 825c944ba..04209a15a 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -37,6 +37,8 @@ #include "spdk/env.h" +#include + static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr, struct nvme_async_event_request *aer); @@ -78,15 +80,21 @@ nvme_ctrlr_set_cc(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cc_regist void spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts) { + char host_id_str[37]; + opts->num_io_queues = DEFAULT_MAX_IO_QUEUES; opts->use_cmb_sqs = true; opts->arb_mechanism = SPDK_NVME_CC_AMS_RR; opts->keep_alive_timeout_ms = 10 * 1000; opts->io_queue_size = DEFAULT_IO_QUEUE_SIZE; - strncpy(opts->hostnqn, DEFAULT_HOSTNQN, sizeof(opts->hostnqn)); opts->io_queue_requests = DEFAULT_IO_QUEUE_REQUESTS; memset(opts->src_addr, 0, sizeof(opts->src_addr)); memset(opts->src_svcid, 0, sizeof(opts->src_svcid)); + memset(opts->host_id, 0, sizeof(opts->host_id)); + memcpy(opts->extended_host_id, g_spdk_nvme_driver->default_extended_host_id, + sizeof(opts->extended_host_id)); + uuid_unparse(opts->extended_host_id, host_id_str); + snprintf(opts->hostnqn, sizeof(opts->hostnqn), "2014-08.org.nvmexpress:uuid:%s", host_id_str); } /** @@ -808,6 +816,76 @@ nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr *ctrlr) return 0; } +static int +nvme_ctrlr_set_host_id(struct spdk_nvme_ctrlr *ctrlr) +{ + struct nvme_completion_poll_status status; + bool all_zeroes; + uint8_t *host_id; + uint32_t host_id_size; + uint32_t i; + int rc; + + if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { + /* + * NVMe-oF sends the host ID during Connect and doesn't allow + * Set Features - Host Identifier after Connect, so we don't need to do anything here. + */ + SPDK_DEBUGLOG(SPDK_TRACE_NVME, "NVMe-oF transport - not sending Set Features - Host ID\n"); + return 0; + } + + if (ctrlr->cdata.ctratt.host_id_exhid_supported) { + SPDK_DEBUGLOG(SPDK_TRACE_NVME, "Using 128-bit extended host identifier\n"); + host_id = ctrlr->opts.extended_host_id; + host_id_size = sizeof(ctrlr->opts.extended_host_id); + } else { + SPDK_DEBUGLOG(SPDK_TRACE_NVME, "Using 64-bit host identifier\n"); + host_id = ctrlr->opts.host_id; + host_id_size = sizeof(ctrlr->opts.host_id); + } + + /* If the user specified an all-zeroes host identifier, don't send the command. */ + all_zeroes = true; + for (i = 0; i < host_id_size; i++) { + if (host_id[i] != 0) { + all_zeroes = false; + break; + } + } + + if (all_zeroes) { + SPDK_DEBUGLOG(SPDK_TRACE_NVME, + "User did not specify host ID - not sending Set Features - Host ID\n"); + return 0; + } + + SPDK_TRACEDUMP(SPDK_TRACE_NVME, "host_id", host_id, host_id_size); + + status.done = false; + rc = nvme_ctrlr_cmd_set_host_id(ctrlr, host_id, host_id_size, nvme_completion_poll_cb, &status); + if (rc != 0) { + SPDK_ERRLOG("Set Features - Host ID failed: %d\n", rc); + return rc; + } + + while (status.done == false) { + spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); + } + if (spdk_nvme_cpl_is_error(&status.cpl)) { + SPDK_WARNLOG("Set Features - Host ID failed: SC 0x%x SCT 0x%x\n", + status.cpl.status.sc, status.cpl.status.sct); + /* + * Treat Set Features - Host ID failure as non-fatal, since the Host ID feature + * is optional. + */ + return 0; + } + + SPDK_DEBUGLOG(SPDK_TRACE_NVME, "Set Features - Host ID was successful\n"); + return 0; +} + static void nvme_ctrlr_destruct_namespaces(struct spdk_nvme_ctrlr *ctrlr) { @@ -1394,6 +1472,10 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr) return -1; } + if (nvme_ctrlr_set_host_id(ctrlr) != 0) { + return -1; + } + return 0; } diff --git a/lib/nvme/nvme_ctrlr_cmd.c b/lib/nvme/nvme_ctrlr_cmd.c index ad22abe99..afa782376 100644 --- a/lib/nvme/nvme_ctrlr_cmd.c +++ b/lib/nvme/nvme_ctrlr_cmd.c @@ -297,7 +297,8 @@ spdk_nvme_ctrlr_cmd_set_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature, int rc; nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); - req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg); + req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg, + true); if (req == NULL) { nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); return -ENOMEM; @@ -325,7 +326,8 @@ spdk_nvme_ctrlr_cmd_get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature, int rc; nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); - req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg); + req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg, + false); if (req == NULL) { nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); return -ENOMEM; @@ -374,6 +376,27 @@ nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr, cb_fn, cb_arg); } +int +nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size, + spdk_nvme_cmd_cb cb_fn, void *cb_arg) +{ + uint32_t cdw11; + + if (host_id_size == 16) { + /* 128-bit extended host identifier */ + cdw11 = 1; + } else if (host_id_size == 8) { + /* 64-bit host identifier */ + cdw11 = 0; + } else { + SPDK_ERRLOG("Invalid host ID size %u\n", host_id_size); + return -EINVAL; + } + + return spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_HOST_IDENTIFIER, cdw11, 0, + host_id, host_id_size, cb_fn, cb_arg); +} + int spdk_nvme_ctrlr_cmd_get_log_page(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page, uint32_t nsid, void *payload, uint32_t payload_size, diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index edf4f282c..26d854f6e 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -110,8 +110,6 @@ #define DEFAULT_ADMIN_QUEUE_REQUESTS (32) #define DEFAULT_IO_QUEUE_REQUESTS (512) -#define DEFAULT_HOSTNQN "nqn.2016-06.io.spdk:init" - enum nvme_payload_type { NVME_PAYLOAD_TYPE_INVALID = 0, @@ -470,6 +468,7 @@ struct nvme_driver { TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs; TAILQ_HEAD(, spdk_nvme_ctrlr) attached_ctrlrs; bool initialized; + uint8_t default_extended_host_id[16]; }; extern struct nvme_driver *g_spdk_nvme_driver; @@ -523,6 +522,8 @@ int nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr, int nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_critical_warning_state state, spdk_nvme_cmd_cb cb_fn, void *cb_arg); +int nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size, + spdk_nvme_cmd_cb cb_fn, void *cb_arg); int nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg); int nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index fe237c661..0afab2705 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -55,7 +55,6 @@ #define NVME_RDMA_TIME_OUT_IN_MS 2000 #define NVME_RDMA_RW_BUFFER_SIZE 131072 -#define NVME_HOST_ID_DEFAULT "12345679890" /* NVME RDMA qpair Resouce Defaults @@ -600,8 +599,9 @@ nvme_rdma_qpair_fabric_connect(struct nvme_rdma_qpair *rqpair) nvmf_data->cntlid = rctrlr->cntlid; } - strncpy((char *)&nvmf_data->hostid, (char *)NVME_HOST_ID_DEFAULT, - strlen((char *)NVME_HOST_ID_DEFAULT)); + SPDK_STATIC_ASSERT(sizeof(nvmf_data->hostid) == sizeof(ctrlr->opts.extended_host_id), + "host ID size mismatch"); + memcpy(nvmf_data->hostid, ctrlr->opts.extended_host_id, sizeof(nvmf_data->hostid)); strncpy((char *)nvmf_data->hostnqn, ctrlr->opts.hostnqn, sizeof(nvmf_data->hostnqn)); strncpy((char *)nvmf_data->subnqn, ctrlr->trid.subnqn, sizeof(nvmf_data->subnqn)); diff --git a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c index 12a74a10b..693c7791c 100644 --- a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c +++ b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c @@ -50,6 +50,8 @@ struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, }; +struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver; + struct spdk_nvme_registers g_ut_nvme_regs = {}; __thread int nvme_thread_ioq_index = -1; @@ -58,6 +60,10 @@ uint32_t set_size = 1; int set_status_cpl = -1; +DEFINE_STUB(nvme_ctrlr_cmd_set_host_id, int, + (struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size, + spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0) + struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid, const struct spdk_nvme_ctrlr_opts *opts, void *devhandle) @@ -1399,6 +1405,11 @@ static void test_ctrlr_opts_set_defaults(void) { struct spdk_nvme_ctrlr_opts opts = {}; + uuid_t uuid; + + CU_ASSERT(uuid_parse("e53e9258-c93b-48b5-be1a-f025af6d232a", uuid) == 0); + SPDK_CU_ASSERT_FATAL(sizeof(uuid) == sizeof(g_spdk_nvme_driver->default_extended_host_id)); + memcpy(g_spdk_nvme_driver->default_extended_host_id, uuid, sizeof(uuid)); spdk_nvme_ctrlr_opts_set_defaults(&opts); CU_ASSERT_EQUAL(opts.num_io_queues, DEFAULT_MAX_IO_QUEUES); @@ -1406,7 +1417,10 @@ test_ctrlr_opts_set_defaults(void) CU_ASSERT_EQUAL(opts.arb_mechanism, SPDK_NVME_CC_AMS_RR); CU_ASSERT_EQUAL(opts.keep_alive_timeout_ms, 10 * 1000); CU_ASSERT_EQUAL(opts.io_queue_size, DEFAULT_IO_QUEUE_SIZE); - CU_ASSERT_STRING_EQUAL(opts.hostnqn, DEFAULT_HOSTNQN); + CU_ASSERT_STRING_EQUAL(opts.hostnqn, + "2014-08.org.nvmexpress:uuid:e53e9258-c93b-48b5-be1a-f025af6d232a"); + CU_ASSERT(memcmp(opts.extended_host_id, g_spdk_nvme_driver->default_extended_host_id, + sizeof(opts.extended_host_id)) == 0); } #if 0 /* TODO: move to PCIe-specific unit test */