From 71ac18d1ad31058aca5acc815c2e633ef56af1e1 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Thu, 28 Feb 2019 07:56:04 -0500 Subject: [PATCH] nvmf: add namespace reservation acquire command support A registrant can obtain a reservation on a namespace by executing acquire command. Acquire command is associated with specific namespace. For now only Acquire and Preempt reservation acquire action is supported, Preempt And Abort will be supported in future. Change-Id: Ifcbb6b414827393ffc266ceada5982b743716321 Signed-off-by: Changpeng Liu Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/436937 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto --- lib/nvmf/ctrlr.c | 1 + lib/nvmf/nvmf_internal.h | 6 ++ lib/nvmf/subsystem.c | 175 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+) diff --git a/lib/nvmf/ctrlr.c b/lib/nvmf/ctrlr.c index e39c2daf1..dcf48975f 100644 --- a/lib/nvmf/ctrlr.c +++ b/lib/nvmf/ctrlr.c @@ -2128,6 +2128,7 @@ spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) case SPDK_NVME_OPC_DATASET_MANAGEMENT: return spdk_nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req); case SPDK_NVME_OPC_RESERVATION_REGISTER: + case SPDK_NVME_OPC_RESERVATION_ACQUIRE: spdk_thread_send_msg(ctrlr->subsys->thread, spdk_nvmf_ns_reservation_request, req); return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; default: diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index dff1f7d67..472459209 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -190,6 +190,12 @@ struct spdk_nvmf_ns { uint32_t gen; /* registrants head */ TAILQ_HEAD(, spdk_nvmf_registrant) registrants; + /* current reservation key */ + uint64_t crkey; + /* reservation type */ + enum spdk_nvme_reservation_type rtype; + /* current reservation holder, only valid if reservation type can only have one holder */ + struct spdk_nvmf_registrant *holder; }; struct spdk_nvmf_qpair { diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index ebb88110a..247430b80 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -1290,6 +1290,30 @@ nvmf_ns_reservation_get_registrant(struct spdk_nvmf_ns *ns, return NULL; } +/* current reservation type is all registrants or not */ +static bool +nvmf_ns_reservation_all_registrants_type(struct spdk_nvmf_ns *ns) +{ + return (ns->rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_ALL_REGS || + ns->rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS); +} + +/* current registrant is reservation holder or not */ +static bool +nvmf_ns_reservation_registrant_is_holder(struct spdk_nvmf_ns *ns, + struct spdk_nvmf_registrant *reg) +{ + if (!reg) { + return false; + } + + if (nvmf_ns_reservation_all_registrants_type(ns)) { + return true; + } + + return (ns->holder == reg); +} + static int nvmf_ns_reservation_add_registrant(struct spdk_nvmf_ns *ns, struct spdk_nvmf_ctrlr *ctrlr, @@ -1321,6 +1345,49 @@ nvmf_ns_reservation_remove_registrant(struct spdk_nvmf_ns *ns, return; } +static uint32_t +nvmf_ns_reservation_remove_registrants_by_key(struct spdk_nvmf_ns *ns, + uint64_t rkey) +{ + struct spdk_nvmf_registrant *reg, *tmp; + uint32_t count = 0; + + TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { + if (reg->rkey == rkey) { + nvmf_ns_reservation_remove_registrant(ns, reg); + count++; + } + } + return count; +} + +static uint32_t +nvmf_ns_reservation_remove_all_other_registrants(struct spdk_nvmf_ns *ns, + struct spdk_nvmf_registrant *reg) +{ + struct spdk_nvmf_registrant *reg_tmp, *reg_tmp2; + uint32_t count = 0; + + TAILQ_FOREACH_SAFE(reg_tmp, &ns->registrants, link, reg_tmp2) { + if (reg_tmp != reg) { + nvmf_ns_reservation_remove_registrant(ns, reg_tmp); + count++; + } + } + return count; +} + +static void +nvmf_ns_reservation_acquire_reservation(struct spdk_nvmf_ns *ns, uint64_t rkey, + enum spdk_nvme_reservation_type rtype, + struct spdk_nvmf_registrant *holder) +{ + ns->rtype = rtype; + ns->crkey = rkey; + assert(ns->holder == NULL); + ns->holder = holder; +} + static void nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns, struct spdk_nvmf_ctrlr *ctrlr, @@ -1411,6 +1478,111 @@ exit: return; } +static void +nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, + struct spdk_nvmf_ctrlr *ctrlr, + struct spdk_nvmf_request *req) +{ + struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; + uint8_t racqa, iekey, rtype; + struct spdk_nvme_reservation_acquire_data key; + struct spdk_nvmf_registrant *reg; + bool all_regs = false; + uint32_t count = 0; + uint8_t status = SPDK_NVME_SC_SUCCESS; + + racqa = cmd->cdw10 & 0x7u; + iekey = (cmd->cdw10 >> 3) & 0x1u; + rtype = (cmd->cdw10 >> 8) & 0xffu; + memcpy(&key, req->data, sizeof(key)); + + SPDK_DEBUGLOG(SPDK_LOG_NVMF, "ACQUIIRE: RACQA %u, IEKEY %u, RTYPE %u, " + "NRKEY 0x%"PRIx64", PRKEY 0x%"PRIx64"\n", + racqa, iekey, rtype, key.crkey, key.prkey); + + if (iekey) { + SPDK_ERRLOG("Ignore existing key field set to 1\n"); + status = SPDK_NVME_SC_INVALID_FIELD; + goto exit; + } + + reg = nvmf_ns_reservation_get_registrant(ns, &ctrlr->hostid); + /* must be registrant and CRKEY must match */ + if (!reg || reg->rkey != key.crkey) { + SPDK_ERRLOG("No registrant or current key doesn't match " + "with existing registrant key\n"); + status = SPDK_NVME_SC_RESERVATION_CONFLICT; + goto exit; + } + + all_regs = nvmf_ns_reservation_all_registrants_type(ns); + + switch (racqa) { + case SPDK_NVME_RESERVE_ACQUIRE: + /* it's not an error for the holder to acquire same reservation type again */ + if (nvmf_ns_reservation_registrant_is_holder(ns, reg) && ns->rtype == rtype) { + /* do nothing */ + } else if (ns->holder == NULL) { + /* fisrt time to acquire the reservation */ + nvmf_ns_reservation_acquire_reservation(ns, key.crkey, rtype, reg); + } else { + SPDK_ERRLOG("Invalid rtype or current registrant is not holder\n"); + status = SPDK_NVME_SC_RESERVATION_CONFLICT; + goto exit; + } + break; + case SPDK_NVME_RESERVE_PREEMPT: + /* no reservation holder */ + if (!ns->holder) { + /* unregister with PRKEY */ + nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey); + break; + } + /* only 1 reservation holder and reservation key is valid */ + if (!all_regs) { + /* preempt itself */ + if (nvmf_ns_reservation_registrant_is_holder(ns, reg) && + ns->crkey == key.prkey) { + ns->rtype = rtype; + break; + } + + if (ns->crkey == key.prkey) { + nvmf_ns_reservation_remove_registrant(ns, ns->holder); + nvmf_ns_reservation_acquire_reservation(ns, key.crkey, rtype, reg); + } else if (key.prkey != 0) { + nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey); + } else { + /* PRKEY is zero */ + SPDK_ERRLOG("Current PRKEY is zero\n"); + status = SPDK_NVME_SC_RESERVATION_CONFLICT; + goto exit; + } + } else { + /* release all other registrants except for the current one */ + if (key.prkey == 0) { + nvmf_ns_reservation_remove_all_other_registrants(ns, reg); + assert(ns->holder == reg); + } else { + count = nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey); + if (count == 0) { + SPDK_ERRLOG("PRKEY doesn't match any registrant\n"); + status = SPDK_NVME_SC_RESERVATION_CONFLICT; + goto exit; + } + } + } + break; + default: + break; + } + +exit: + req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; + req->rsp->nvme_cpl.status.sc = status; + return; +} + static void spdk_nvmf_ns_reservation_complete(void *ctx) { @@ -1437,6 +1609,9 @@ spdk_nvmf_ns_reservation_request(void *ctx) case SPDK_NVME_OPC_RESERVATION_REGISTER: nvmf_ns_reservation_register(ns, ctrlr, req); break; + case SPDK_NVME_OPC_RESERVATION_ACQUIRE: + nvmf_ns_reservation_acquire(ns, ctrlr, req); + break; default: break; }