From bc1d0b91b50d147d9225452fefabb0a4cd0e8186 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Wed, 27 Feb 2019 21:50:03 -0500 Subject: [PATCH] nvmf: add namespace reservation register command support Reservations can be used by two or more hosts to coordinate acccess to a shared namespace, host must register to a namespace prior to establishing a reservation. Unregistering by a host may cause a reservation release, this feature will be supported after reservation acquire patch. Change-Id: Id44aa1f82f30d9ecc5999a2a9a7c20b2af77774a Signed-off-by: Changpeng Liu Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/436936 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto --- lib/nvmf/ctrlr.c | 3 + lib/nvmf/nvmf_internal.h | 12 ++ lib/nvmf/subsystem.c | 174 ++++++++++++++++++++++++++ test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c | 2 + test/unit/lib/nvmf/tcp.c/tcp_ut.c | 2 + 5 files changed, 193 insertions(+) diff --git a/lib/nvmf/ctrlr.c b/lib/nvmf/ctrlr.c index 19b505731..e39c2daf1 100644 --- a/lib/nvmf/ctrlr.c +++ b/lib/nvmf/ctrlr.c @@ -2127,6 +2127,9 @@ spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) return spdk_nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req); case SPDK_NVME_OPC_DATASET_MANAGEMENT: return spdk_nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req); + case SPDK_NVME_OPC_RESERVATION_REGISTER: + spdk_thread_send_msg(ctrlr->subsys->thread, spdk_nvmf_ns_reservation_request, req); + return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; default: return spdk_nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req); } diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index 812f0d09f..dff1f7d67 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -171,6 +171,13 @@ struct spdk_nvmf_request { TAILQ_ENTRY(spdk_nvmf_request) link; }; +struct spdk_nvmf_registrant { + TAILQ_ENTRY(spdk_nvmf_registrant) link; + struct spdk_uuid hostid; + /* Registration key */ + uint64_t rkey; +}; + struct spdk_nvmf_ns { uint32_t nsid; struct spdk_nvmf_subsystem *subsystem; @@ -179,6 +186,10 @@ struct spdk_nvmf_ns { struct spdk_nvmf_ns_opts opts; /* reservation notificaton mask */ uint32_t mask; + /* generation code */ + uint32_t gen; + /* registrants head */ + TAILQ_HEAD(, spdk_nvmf_registrant) registrants; }; struct spdk_nvmf_qpair { @@ -327,6 +338,7 @@ void spdk_nvmf_subsystem_remove_ctrlr(struct spdk_nvmf_subsystem *subsystem, struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem *subsystem, uint16_t cntlid); int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr); +void spdk_nvmf_ns_reservation_request(void *ctx); /* * Abort aer is sent on a per controller basis and sends a completion for the aer to the host. diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index 4df1c56a7..ebb88110a 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -883,6 +883,7 @@ static int _spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid) { struct spdk_nvmf_ns *ns; + struct spdk_nvmf_registrant *reg, *reg_tmp; assert(subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSED || subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE); @@ -903,6 +904,10 @@ _spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t n subsystem->ns[nsid - 1] = NULL; + TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, reg_tmp) { + TAILQ_REMOVE(&ns->registrants, reg, link); + free(reg); + } spdk_bdev_module_release_bdev(ns->bdev); spdk_bdev_close(ns->desc); free(ns); @@ -1079,6 +1084,7 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd } subsystem->ns[opts.nsid - 1] = ns; ns->nsid = opts.nsid; + TAILQ_INIT(&ns->registrants); SPDK_DEBUGLOG(SPDK_LOG_NVMF, "Subsystem %s: bdev %s assigned nsid %" PRIu32 "\n", spdk_nvmf_subsystem_get_nqn(subsystem), @@ -1268,3 +1274,171 @@ spdk_nvmf_subsystem_get_max_namespaces(const struct spdk_nvmf_subsystem *subsyst { return subsystem->max_allowed_nsid; } + +static struct spdk_nvmf_registrant * +nvmf_ns_reservation_get_registrant(struct spdk_nvmf_ns *ns, + struct spdk_uuid *uuid) +{ + struct spdk_nvmf_registrant *reg, *tmp; + + TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { + if (spdk_uuid_compare(®->hostid, uuid) == 0) { + return reg; + } + } + + return NULL; +} + +static int +nvmf_ns_reservation_add_registrant(struct spdk_nvmf_ns *ns, + struct spdk_nvmf_ctrlr *ctrlr, + uint64_t nrkey) +{ + struct spdk_nvmf_registrant *reg; + + reg = calloc(1, sizeof(*reg)); + if (!reg) { + return -ENOMEM; + } + + reg->rkey = nrkey; + /* set hostid for the registrant */ + spdk_uuid_copy(®->hostid, &ctrlr->hostid); + TAILQ_INSERT_TAIL(&ns->registrants, reg, link); + ns->gen++; + + return 0; +} + +static void +nvmf_ns_reservation_remove_registrant(struct spdk_nvmf_ns *ns, + struct spdk_nvmf_registrant *reg) +{ + TAILQ_REMOVE(&ns->registrants, reg, link); + free(reg); + ns->gen++; + return; +} + +static void +nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns, + struct spdk_nvmf_ctrlr *ctrlr, + struct spdk_nvmf_request *req) +{ + struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; + uint8_t rrega, iekey, cptpl; + struct spdk_nvme_reservation_register_data key; + struct spdk_nvmf_registrant *reg; + uint8_t status = SPDK_NVME_SC_SUCCESS; + int rc; + + rrega = cmd->cdw10 & 0x7u; + iekey = (cmd->cdw10 >> 3) & 0x1u; + cptpl = (cmd->cdw10 >> 30) & 0x3u; + memcpy(&key, req->data, sizeof(key)); + + SPDK_DEBUGLOG(SPDK_LOG_NVMF, "REGISTER: RREGA %u, IEKEY %u, CPTPL %u, " + "NRKEY 0x%"PRIx64", NRKEY 0x%"PRIx64"\n", + rrega, iekey, cptpl, key.crkey, key.nrkey); + + /* TODO: doesn't support for now */ + if (cptpl == SPDK_NVME_RESERVE_PTPL_PERSIST_POWER_LOSS) { + SPDK_ERRLOG("Can't change persist through power loss for now\n"); + status = SPDK_NVME_SC_INVALID_FIELD; + goto exit; + } + + /* current Host Identifier has registrant or not */ + reg = nvmf_ns_reservation_get_registrant(ns, &ctrlr->hostid); + + switch (rrega) { + case SPDK_NVME_RESERVE_REGISTER_KEY: + if (!reg) { + /* register new controller */ + if (key.nrkey == 0) { + SPDK_ERRLOG("Can't register zeroed new key\n"); + status = SPDK_NVME_SC_INVALID_FIELD; + goto exit; + } + rc = nvmf_ns_reservation_add_registrant(ns, ctrlr, key.nrkey); + if (rc < 0) { + status = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; + goto exit; + } + } else { + /* register with same key is not an error */ + if (reg->rkey != key.nrkey) { + SPDK_ERRLOG("The same host already register a " + "key with 0x%"PRIx64"\n", + reg->rkey); + status = SPDK_NVME_SC_RESERVATION_CONFLICT; + goto exit; + } + } + break; + case SPDK_NVME_RESERVE_UNREGISTER_KEY: + if (!reg || (!iekey && reg->rkey != key.crkey)) { + SPDK_ERRLOG("No registrant or current key doesn't match " + "with existing registrant key\n"); + status = SPDK_NVME_SC_RESERVATION_CONFLICT; + goto exit; + } + nvmf_ns_reservation_remove_registrant(ns, reg); + break; + case SPDK_NVME_RESERVE_REPLACE_KEY: + if (!reg || (!iekey && reg->rkey != key.crkey)) { + SPDK_ERRLOG("No registrant or current key doesn't match " + "with existing registrant key\n"); + status = SPDK_NVME_SC_RESERVATION_CONFLICT; + goto exit; + } + if (key.nrkey == 0) { + SPDK_ERRLOG("Can't register zeroed new key\n"); + status = SPDK_NVME_SC_INVALID_FIELD; + goto exit; + } + reg->rkey = key.nrkey; + break; + default: + status = SPDK_NVME_SC_INVALID_FIELD; + goto exit; + } + +exit: + req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; + req->rsp->nvme_cpl.status.sc = status; + return; +} + +static void +spdk_nvmf_ns_reservation_complete(void *ctx) +{ + struct spdk_nvmf_request *req = ctx; + + spdk_nvmf_request_complete(req); +} + +void +spdk_nvmf_ns_reservation_request(void *ctx) +{ + struct spdk_nvmf_request *req = (struct spdk_nvmf_request *)ctx; + struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; + struct spdk_nvmf_poll_group *group = req->qpair->group; + struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; + uint32_t nsid; + struct spdk_nvmf_ns *ns; + + nsid = cmd->nsid; + ns = _spdk_nvmf_subsystem_get_ns(ctrlr->subsys, nsid); + assert(ns != NULL); + + switch (cmd->opc) { + case SPDK_NVME_OPC_RESERVATION_REGISTER: + nvmf_ns_reservation_register(ns, ctrlr, req); + break; + default: + break; + } + spdk_thread_send_msg(group->thread, spdk_nvmf_ns_reservation_complete, req); +} diff --git a/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c b/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c index 2b8ce4a70..daae322d8 100644 --- a/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c +++ b/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c @@ -156,6 +156,8 @@ DEFINE_STUB(spdk_nvmf_transport_req_complete, (struct spdk_nvmf_request *req), 0); +DEFINE_STUB_V(spdk_nvmf_ns_reservation_request, (void *ctx)); + int spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) { diff --git a/test/unit/lib/nvmf/tcp.c/tcp_ut.c b/test/unit/lib/nvmf/tcp.c/tcp_ut.c index 396a509a3..7b4ea75e9 100644 --- a/test/unit/lib/nvmf/tcp.c/tcp_ut.c +++ b/test/unit/lib/nvmf/tcp.c/tcp_ut.c @@ -158,6 +158,8 @@ DEFINE_STUB(spdk_nvmf_transport_req_complete, (struct spdk_nvmf_request *req), 0); +DEFINE_STUB_V(spdk_nvmf_ns_reservation_request, (void *ctx)); + struct spdk_trace_histories *g_trace_histories; struct spdk_bdev {