ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme: add controller option to specify host ID

Browse Source 
A random host ID is generated per SPDK application startup if the user
doesn't specify a host ID during controller startup.

This also changes the default host NQN for NVMe-oF connections to a
random UUID NQN based on the host ID.

Change-Id: Ib0f70dd63e53087716842b412a1f134a9991d4da
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-on: https://review.gerrithub.io/380528
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
This commit is contained in:
Daniel Verkamp 2017-09-27 10:52:47 -07:00 committed by Jim Harris
parent 04124278b9
commit 8acc4ae5da
9 changed files with 192 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -43,6 +43,12 @@ The HotplugEnable option in `[Nvme]` sections of the configuration file is now
The NVMe library now includes a function spdk_nvme_ns_get_ctrlr which returns the
NVMe Controller associated with a given namespace.
The NVMe library now allows the user to specify a host identifier when attaching
to a controller. The host identifier is used as part of the Reservations feature,
as well as in the NVMe-oF Connect command. The default host ID is also now a
randomly-generated UUID, and the default host NQN uses the host ID to generate
a UUID-based NQN.
### NVMe-oF Target (nvmf_tgt)
The NVMe-oF target no longer requires any in capsule data buffers to run, and

123
examples/nvme/reserve/reserve.c
View File

@ -33,8 +33,10 @@
#include "spdk/stdinc.h"
#include "spdk/endian.h"
#include "spdk/nvme.h"
#include "spdk/env.h"
#include "spdk/log.h"
#define MAX_DEVS 64
@ -52,44 +54,22 @@ static int num_devs = 0;
static int outstanding_commands;
static int reserve_command_result;
static int set_feature_result;
struct feature {
uint32_t result;
bool valid;
};
static struct feature features[256];
static bool get_host_id_successful;
#define HOST_ID 0xABABABABCDCDCDCD
#define EXT_HOST_ID ((uint8_t[]){0x0f, 0x97, 0xcd, 0x74, 0x8c, 0x80, 0x41, 0x42, \
0x99, 0x0f, 0x65, 0xc4, 0xf0, 0x39, 0x24, 0x20})
#define CR_KEY 0xDEADBEAF5A5A5A5B
static void
get_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
{
struct feature *feature = cb_arg;
int fid = feature - features;
if (spdk_nvme_cpl_is_error(cpl)) {
fprintf(stdout, "get_feature(0x%02X) failed\n", fid);
fprintf(stdout, "Get Features - Host Identifier failed\n");
get_host_id_successful = false;
} else {
feature->result = cpl->cdw0;
feature->valid = true;
}
outstanding_commands--;
}
static void
set_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
{
struct feature *feature = cb_arg;
int fid = feature - features;
if (spdk_nvme_cpl_is_error(cpl)) {
fprintf(stdout, "set_feature(0x%02X) failed\n", fid);
set_feature_result = -1;
} else {
set_feature_result = 0;
get_host_id_successful = true;
}
outstanding_commands--;
}
@ -98,81 +78,40 @@ static int
get_host_identifier(struct spdk_nvme_ctrlr *ctrlr)
{
int ret;
uint64_t *host_id;
struct spdk_nvme_cmd cmd = {};
uint8_t host_id[16];
uint32_t host_id_size;
uint32_t cdw11;
cmd.opc = SPDK_NVME_OPC_GET_FEATURES;
cmd.cdw10 = SPDK_NVME_FEAT_HOST_IDENTIFIER;
host_id = spdk_dma_zmalloc(sizeof(*host_id), 0x1000, NULL);
if (!host_id) {
fprintf(stderr, "Host_ID DMA Buffer Allocation Failed\n");
return -1;
if (spdk_nvme_ctrlr_get_data(ctrlr)->ctratt.host_id_exhid_supported) {
host_id_size = 16;
cdw11 = 1;
printf("Using 128-bit extended host identifier\n");
} else {
host_id_size = 8;
cdw11 = 0;
printf("Using 64-bit host identifier\n");
}
outstanding_commands = 0;
ret = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, host_id, sizeof(*host_id),
get_feature_completion, &features[SPDK_NVME_FEAT_HOST_IDENTIFIER]);
ret = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_HOST_IDENTIFIER, cdw11, host_id,
host_id_size,
get_feature_completion, NULL);
if (ret) {
fprintf(stdout, "Get Feature: Failed\n");
spdk_dma_free(host_id);
return -1;
}
outstanding_commands++;
get_host_id_successful = false;
while (outstanding_commands) {
spdk_nvme_ctrlr_process_admin_completions(ctrlr);
}
if (features[SPDK_NVME_FEAT_HOST_IDENTIFIER].valid) {
fprintf(stdout, "Get Feature: Host Identifier 0x%" PRIx64 "\n", *host_id);
if (get_host_id_successful) {
spdk_trace_dump(stdout, "Get Feature: Host Identifier:", host_id, host_id_size);
}
spdk_dma_free(host_id);
return 0;
}
static int
set_host_identifier(struct spdk_nvme_ctrlr *ctrlr)
{
int ret;
uint64_t *host_id;
struct spdk_nvme_cmd cmd = {};
cmd.opc = SPDK_NVME_OPC_SET_FEATURES;
cmd.cdw10 = SPDK_NVME_FEAT_HOST_IDENTIFIER;
host_id = spdk_dma_zmalloc(sizeof(*host_id), 0x1000, NULL);
if (!host_id) {
fprintf(stderr, "Host_ID DMA Buffer Allocation Failed\n");
return -1;
}
*host_id = HOST_ID;
outstanding_commands = 0;
set_feature_result = -1;
fprintf(stdout, "Set Feature: Host Identifier 0x%" PRIx64 "\n", *host_id);
ret = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, host_id, sizeof(*host_id),
set_feature_completion, &features[SPDK_NVME_FEAT_HOST_IDENTIFIER]);
if (ret) {
fprintf(stdout, "Set Feature: Failed\n");
spdk_dma_free(host_id);
return -1;
}
outstanding_commands++;
while (outstanding_commands) {
spdk_nvme_ctrlr_process_admin_completions(ctrlr);
}
if (set_feature_result)
fprintf(stdout, "Set Feature: Host Identifier Failed\n");
spdk_dma_free(host_id);
return 0;
}
@ -369,7 +308,6 @@ reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
if (!cdata->oncs.reservations)
return;
set_host_identifier(ctrlr);
get_host_identifier(ctrlr);
/* tested 1 namespace */
@ -383,6 +321,15 @@ static bool
probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
struct spdk_nvme_ctrlr_opts *opts)
{
/*
* Provide both 64-bit and 128-bit host identifiers.
*
* The NVMe library will choose which one to use based on whether the controller
* supports extended host identifiers.
*/
to_le64(opts->host_id, HOST_ID);
memcpy(opts->extended_host_id, EXT_HOST_ID, sizeof(opts->extended_host_id));
return true;
}

14
include/spdk/nvme.h
View File

@ -130,6 +130,20 @@ struct spdk_nvme_ctrlr_opts {
* specified.
*/
char src_svcid[SPDK_NVMF_TRSVCID_MAX_LEN + 1];
/**
* The host identifier to use when connecting to controllers with 64-bit host ID support.
*
* Set to all zeroes to specify that no host ID should be provided to the controller.
*/
uint8_t host_id[8];
/**
* The host identifier to use when connecting to controllers with extended (128-bit) host ID support.
*
* Set to all zeroes to specify that no host ID should be provided to the controller.
*/
uint8_t extended_host_id[16];
};
/**

8
lib/nvme/nvme.c
View File

@ -34,6 +34,8 @@
#include "spdk/nvmf_spec.h"
#include "nvme_internal.h"
#include <uuid/uuid.h>
#define SPDK_NVME_DRIVER_NAME "spdk_nvme_driver"
struct nvme_driver *g_spdk_nvme_driver;
@ -227,6 +229,7 @@ nvme_robust_mutex_init_shared(pthread_mutex_t *mtx)
static int
nvme_driver_init(void)
{
uuid_t host_id;
int ret = 0;
/* Any socket ID */
int socket_id = -1;
@ -300,6 +303,11 @@ nvme_driver_init(void)
TAILQ_INIT(&g_spdk_nvme_driver->init_ctrlrs);
TAILQ_INIT(&g_spdk_nvme_driver->attached_ctrlrs);
SPDK_STATIC_ASSERT(sizeof(host_id) == sizeof(g_spdk_nvme_driver->default_extended_host_id),
"host ID size mismatch");
uuid_generate(host_id);
memcpy(g_spdk_nvme_driver->default_extended_host_id, host_id, sizeof(host_id));
nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
return ret;

84
lib/nvme/nvme_ctrlr.c
View File

@ -37,6 +37,8 @@
#include "spdk/env.h"
#include <uuid/uuid.h>
static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr,
struct nvme_async_event_request *aer);
@ -78,15 +80,21 @@ nvme_ctrlr_set_cc(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cc_regist
void
spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts)
{
char host_id_str[37];
opts->num_io_queues = DEFAULT_MAX_IO_QUEUES;
opts->use_cmb_sqs = true;
opts->arb_mechanism = SPDK_NVME_CC_AMS_RR;
opts->keep_alive_timeout_ms = 10 * 1000;
opts->io_queue_size = DEFAULT_IO_QUEUE_SIZE;
strncpy(opts->hostnqn, DEFAULT_HOSTNQN, sizeof(opts->hostnqn));
opts->io_queue_requests = DEFAULT_IO_QUEUE_REQUESTS;
memset(opts->src_addr, 0, sizeof(opts->src_addr));
memset(opts->src_svcid, 0, sizeof(opts->src_svcid));
memset(opts->host_id, 0, sizeof(opts->host_id));
memcpy(opts->extended_host_id, g_spdk_nvme_driver->default_extended_host_id,
sizeof(opts->extended_host_id));
uuid_unparse(opts->extended_host_id, host_id_str);
snprintf(opts->hostnqn, sizeof(opts->hostnqn), "2014-08.org.nvmexpress:uuid:%s", host_id_str);
}
/**
@ -808,6 +816,76 @@ nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr *ctrlr)
return 0;
}
static int
nvme_ctrlr_set_host_id(struct spdk_nvme_ctrlr *ctrlr)
{
struct nvme_completion_poll_status status;
bool all_zeroes;
uint8_t *host_id;
uint32_t host_id_size;
uint32_t i;
int rc;
if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
/*
* NVMe-oF sends the host ID during Connect and doesn't allow
* Set Features - Host Identifier after Connect, so we don't need to do anything here.
*/
SPDK_DEBUGLOG(SPDK_TRACE_NVME, "NVMe-oF transport - not sending Set Features - Host ID\n");
return 0;
}
if (ctrlr->cdata.ctratt.host_id_exhid_supported) {
SPDK_DEBUGLOG(SPDK_TRACE_NVME, "Using 128-bit extended host identifier\n");
host_id = ctrlr->opts.extended_host_id;
host_id_size = sizeof(ctrlr->opts.extended_host_id);
} else {
SPDK_DEBUGLOG(SPDK_TRACE_NVME, "Using 64-bit host identifier\n");
host_id = ctrlr->opts.host_id;
host_id_size = sizeof(ctrlr->opts.host_id);
}
/* If the user specified an all-zeroes host identifier, don't send the command. */
all_zeroes = true;
for (i = 0; i < host_id_size; i++) {
if (host_id[i] != 0) {
all_zeroes = false;
break;
}
}
if (all_zeroes) {
SPDK_DEBUGLOG(SPDK_TRACE_NVME,
"User did not specify host ID - not sending Set Features - Host ID\n");
return 0;
}
SPDK_TRACEDUMP(SPDK_TRACE_NVME, "host_id", host_id, host_id_size);
status.done = false;
rc = nvme_ctrlr_cmd_set_host_id(ctrlr, host_id, host_id_size, nvme_completion_poll_cb, &status);
if (rc != 0) {
SPDK_ERRLOG("Set Features - Host ID failed: %d\n", rc);
return rc;
}
while (status.done == false) {
spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
}
if (spdk_nvme_cpl_is_error(&status.cpl)) {
SPDK_WARNLOG("Set Features - Host ID failed: SC 0x%x SCT 0x%x\n",
status.cpl.status.sc, status.cpl.status.sct);
/*
* Treat Set Features - Host ID failure as non-fatal, since the Host ID feature
* is optional.
*/
return 0;
}
SPDK_DEBUGLOG(SPDK_TRACE_NVME, "Set Features - Host ID was successful\n");
return 0;
}
static void
nvme_ctrlr_destruct_namespaces(struct spdk_nvme_ctrlr *ctrlr)
{
@ -1394,6 +1472,10 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr)
return -1;
}
if (nvme_ctrlr_set_host_id(ctrlr) != 0) {
return -1;
}
return 0;
}

27
lib/nvme/nvme_ctrlr_cmd.c
View File

@ -297,7 +297,8 @@ spdk_nvme_ctrlr_cmd_set_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature,
int rc;
nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg,
true);
if (req == NULL) {
nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
return -ENOMEM;
@ -325,7 +326,8 @@ spdk_nvme_ctrlr_cmd_get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature,
int rc;
nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
req = nvme_allocate_request_user_copy(ctrlr->adminq, payload, payload_size, cb_fn, cb_arg,
false);
if (req == NULL) {
nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
return -ENOMEM;
@ -374,6 +376,27 @@ nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
cb_fn, cb_arg);
}
int
nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
spdk_nvme_cmd_cb cb_fn, void *cb_arg)
{
uint32_t cdw11;
if (host_id_size == 16) {
/* 128-bit extended host identifier */
cdw11 = 1;
} else if (host_id_size == 8) {
/* 64-bit host identifier */
cdw11 = 0;
} else {
SPDK_ERRLOG("Invalid host ID size %u\n", host_id_size);
return -EINVAL;
}
return spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_HOST_IDENTIFIER, cdw11, 0,
host_id, host_id_size, cb_fn, cb_arg);
}
int
spdk_nvme_ctrlr_cmd_get_log_page(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page,
uint32_t nsid, void *payload, uint32_t payload_size,

5
lib/nvme/nvme_internal.h
View File

@ -110,8 +110,6 @@
#define DEFAULT_ADMIN_QUEUE_REQUESTS (32)
#define DEFAULT_IO_QUEUE_REQUESTS (512)
#define DEFAULT_HOSTNQN "nqn.2016-06.io.spdk:init"
enum nvme_payload_type {
NVME_PAYLOAD_TYPE_INVALID = 0,
@ -470,6 +468,7 @@ struct nvme_driver {
TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs;
TAILQ_HEAD(, spdk_nvme_ctrlr) attached_ctrlrs;
bool initialized;
uint8_t default_extended_host_id[16];
};
extern struct nvme_driver *g_spdk_nvme_driver;
@ -523,6 +522,8 @@ int nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr,
int nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
union spdk_nvme_critical_warning_state state,
spdk_nvme_cmd_cb cb_fn, void *cb_arg);
int nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
spdk_nvme_cmd_cb cb_fn, void *cb_arg);
int nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
int nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,

6
lib/nvme/nvme_rdma.c
View File

@ -55,7 +55,6 @@
#define NVME_RDMA_TIME_OUT_IN_MS 2000
#define NVME_RDMA_RW_BUFFER_SIZE 131072
#define NVME_HOST_ID_DEFAULT "12345679890"
/*
NVME RDMA qpair Resouce Defaults
@ -600,8 +599,9 @@ nvme_rdma_qpair_fabric_connect(struct nvme_rdma_qpair *rqpair)
nvmf_data->cntlid = rctrlr->cntlid;
}
strncpy((char *)&nvmf_data->hostid, (char *)NVME_HOST_ID_DEFAULT,
strlen((char *)NVME_HOST_ID_DEFAULT));
SPDK_STATIC_ASSERT(sizeof(nvmf_data->hostid) == sizeof(ctrlr->opts.extended_host_id),
"host ID size mismatch");
memcpy(nvmf_data->hostid, ctrlr->opts.extended_host_id, sizeof(nvmf_data->hostid));
strncpy((char *)nvmf_data->hostnqn, ctrlr->opts.hostnqn, sizeof(nvmf_data->hostnqn));
strncpy((char *)nvmf_data->subnqn, ctrlr->trid.subnqn, sizeof(nvmf_data->subnqn));

16
test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c
View File

@ -50,6 +50,8 @@ struct nvme_driver _g_nvme_driver = {
.lock = PTHREAD_MUTEX_INITIALIZER,
};
struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver;
struct spdk_nvme_registers g_ut_nvme_regs = {};
__thread int nvme_thread_ioq_index = -1;
@ -58,6 +60,10 @@ uint32_t set_size = 1;
int set_status_cpl = -1;
DEFINE_STUB(nvme_ctrlr_cmd_set_host_id, int,
(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0)
struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
const struct spdk_nvme_ctrlr_opts *opts,
void *devhandle)
@ -1399,6 +1405,11 @@ static void
test_ctrlr_opts_set_defaults(void)
{
struct spdk_nvme_ctrlr_opts opts = {};
uuid_t uuid;
CU_ASSERT(uuid_parse("e53e9258-c93b-48b5-be1a-f025af6d232a", uuid) == 0);
SPDK_CU_ASSERT_FATAL(sizeof(uuid) == sizeof(g_spdk_nvme_driver->default_extended_host_id));
memcpy(g_spdk_nvme_driver->default_extended_host_id, uuid, sizeof(uuid));
spdk_nvme_ctrlr_opts_set_defaults(&opts);
CU_ASSERT_EQUAL(opts.num_io_queues, DEFAULT_MAX_IO_QUEUES);
@ -1406,7 +1417,10 @@ test_ctrlr_opts_set_defaults(void)
CU_ASSERT_EQUAL(opts.arb_mechanism, SPDK_NVME_CC_AMS_RR);
CU_ASSERT_EQUAL(opts.keep_alive_timeout_ms, 10 * 1000);
CU_ASSERT_EQUAL(opts.io_queue_size, DEFAULT_IO_QUEUE_SIZE);
CU_ASSERT_STRING_EQUAL(opts.hostnqn, DEFAULT_HOSTNQN);
CU_ASSERT_STRING_EQUAL(opts.hostnqn,
"2014-08.org.nvmexpress:uuid:e53e9258-c93b-48b5-be1a-f025af6d232a");
CU_ASSERT(memcmp(opts.extended_host_id, g_spdk_nvme_driver->default_extended_host_id,
sizeof(opts.extended_host_id)) == 0);
}
#if 0 /* TODO: move to PCIe-specific unit test */