From 78bfb2a1d0d9a5e32caf19c45eca36f65e90d0c8 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Fri, 1 Mar 2019 03:58:57 -0500 Subject: [PATCH] nvmf: generate reservation notification log pages A host can use the Asynchronous Event Command to be notified of the presense of one or more avaiable reservation notification log pages. A reservation notificaton log page should be created whenever an unmasked reservation notification occurs. Change-Id: I8b83e5319725286dd0a5efc1b22d8ac4673e31e1 Signed-off-by: Changpeng Liu Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/439931 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- lib/nvmf/ctrlr.c | 58 +++++++ lib/nvmf/nvmf_internal.h | 14 ++ lib/nvmf/subsystem.c | 162 +++++++++++++++++- test/unit/lib/nvmf/subsystem.c/subsystem_ut.c | 48 +++++- 4 files changed, 280 insertions(+), 2 deletions(-) diff --git a/lib/nvmf/ctrlr.c b/lib/nvmf/ctrlr.c index eae11c99c..8feb7bb86 100644 --- a/lib/nvmf/ctrlr.c +++ b/lib/nvmf/ctrlr.c @@ -269,6 +269,7 @@ spdk_nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem, return NULL; } + TAILQ_INIT(&ctrlr->log_head); ctrlr->subsys = subsystem; ctrlr->thread = req->qpair->group->thread; @@ -361,8 +362,14 @@ static void _spdk_nvmf_ctrlr_destruct(void *ctx) { struct spdk_nvmf_ctrlr *ctrlr = ctx; + struct spdk_nvmf_reservation_log *log, *log_tmp; spdk_nvmf_ctrlr_stop_keep_alive_timer(ctrlr); + + TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { + TAILQ_REMOVE(&ctrlr->log_head, log, link); + free(log); + } free(ctrlr); } @@ -2080,6 +2087,57 @@ spdk_nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr) ctrlr->aer_req = NULL; } +void +spdk_nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr, + struct spdk_nvmf_ns *ns, + enum spdk_nvme_reservation_notification_log_page_type type) +{ + struct spdk_nvmf_reservation_log *log; + + switch (type) { + case SPDK_NVME_RESERVATION_LOG_PAGE_EMPTY: + return; + case SPDK_NVME_REGISTRATION_PREEMPTED: + if (ns->mask & SPDK_NVME_REGISTRATION_PREEMPTED_MASK) { + return; + } + break; + case SPDK_NVME_RESERVATION_RELEASED: + if (ns->mask & SPDK_NVME_RESERVATION_RELEASED_MASK) { + return; + } + break; + case SPDK_NVME_RESERVATION_PREEMPTED: + if (ns->mask & SPDK_NVME_RESERVATION_PREEMPTED_MASK) { + return; + } + break; + default: + return; + } + + ctrlr->log_page_count++; + + /* Maximum number of queued log pages is 255 */ + if (ctrlr->num_avail_log_pages == 0xff) { + log = TAILQ_LAST(&ctrlr->log_head, log_page_head); + log->log.log_page_count = ctrlr->log_page_count; + return; + } + + log = calloc(1, sizeof(*log)); + if (!log) { + SPDK_ERRLOG("Alloc log page failed, ignore the log\n"); + return; + } + + log->log.log_page_count = ctrlr->log_page_count; + log->log.type = type; + log->log.num_avail_log_pages = ctrlr->num_avail_log_pages++; + log->log.nsid = ns->nsid; + TAILQ_INSERT_TAIL(&ctrlr->log_head, log, link); +} + /* Check from subsystem poll group's namespace information data structure */ static bool nvmf_ns_info_ctrlr_is_registrant(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index e21b884a5..4989ab418 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -241,6 +241,14 @@ struct spdk_nvmf_ctrlr_feat { union spdk_nvme_feat_keep_alive_timer keep_alive_timer; }; +/* + * NVMf reservation notificaton log page. + */ +struct spdk_nvmf_reservation_log { + struct spdk_nvme_reservation_notification_log log; + TAILQ_ENTRY(spdk_nvmf_reservation_log) link; +}; + /* * This structure represents an NVMe-oF controller, * which is like a "session" in networking terms. @@ -268,6 +276,9 @@ struct spdk_nvmf_ctrlr { uint16_t changed_ns_list_count; struct spdk_nvme_ns_list changed_ns_list; + uint64_t log_page_count; + uint8_t num_avail_log_pages; + TAILQ_HEAD(log_page_head, spdk_nvmf_reservation_log) log_head; /* Time to trigger keep-alive--poller_time = now_tick + period */ uint64_t last_keep_alive_tick; @@ -360,6 +371,9 @@ struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem uint16_t cntlid); int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr); void spdk_nvmf_ns_reservation_request(void *ctx); +void spdk_nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr, + struct spdk_nvmf_ns *ns, + enum spdk_nvme_reservation_notification_log_page_type type); /* * Abort aer is sent on a per controller basis and sends a completion for the aer to the host. diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index e2fdcdd14..713a72ba6 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -1303,6 +1303,87 @@ nvmf_ns_reservation_get_registrant(struct spdk_nvmf_ns *ns, return NULL; } +/* Generate reservation notice log to registered HostID controllers */ +static void +nvmf_subsystem_gen_ctrlr_notification(struct spdk_nvmf_subsystem *subsystem, + struct spdk_nvmf_ns *ns, + struct spdk_uuid *hostid_list, + uint32_t num_hostid, + enum spdk_nvme_reservation_notification_log_page_type type) +{ + struct spdk_nvmf_ctrlr *ctrlr; + uint32_t i; + + for (i = 0; i < num_hostid; i++) { + TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { + if (!spdk_uuid_compare(&ctrlr->hostid, &hostid_list[i])) { + spdk_nvmf_ctrlr_reservation_notice_log(ctrlr, ns, type); + } + } + } +} + +/* Get all registrants' hostid other than the controller who issued the command */ +static uint32_t +nvmf_ns_reservation_get_all_other_hostid(struct spdk_nvmf_ns *ns, + struct spdk_uuid *hostid_list, + uint32_t max_num_hostid, + struct spdk_uuid *current_hostid) +{ + struct spdk_nvmf_registrant *reg, *tmp; + uint32_t num_hostid = 0; + + TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { + if (spdk_uuid_compare(®->hostid, current_hostid)) { + if (num_hostid == max_num_hostid) { + assert(false); + return max_num_hostid; + } + hostid_list[num_hostid++] = reg->hostid; + } + } + + return num_hostid; +} + +/* Calculate the unregistered HostID list according to list + * prior to execute preempt command and list after executing + * preempt command. + */ +static uint32_t +nvmf_ns_reservation_get_unregistered_hostid(struct spdk_uuid *old_hostid_list, + uint32_t old_num_hostid, + struct spdk_uuid *remaining_hostid_list, + uint32_t remaining_num_hostid) +{ + struct spdk_uuid temp_hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS]; + uint32_t i, j, num_hostid = 0; + bool found; + + if (!remaining_num_hostid) { + return old_num_hostid; + } + + for (i = 0; i < old_num_hostid; i++) { + found = false; + for (j = 0; j < remaining_num_hostid; j++) { + if (!spdk_uuid_compare(&old_hostid_list[i], &remaining_hostid_list[j])) { + found = true; + break; + } + } + if (!found) { + spdk_uuid_copy(&temp_hostid_list[num_hostid++], &old_hostid_list[i]); + } + } + + if (num_hostid) { + memcpy(old_hostid_list, temp_hostid_list, sizeof(struct spdk_uuid) * num_hostid); + } + + return num_hostid; +} + /* current reservation type is all registrants or not */ static bool nvmf_ns_reservation_all_registrants_type(struct spdk_nvmf_ns *ns) @@ -1452,11 +1533,13 @@ nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns, struct spdk_nvmf_request *req) { struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; - uint8_t rrega, iekey, cptpl; + uint8_t rrega, iekey, cptpl, rtype; struct spdk_nvme_reservation_register_data key; struct spdk_nvmf_registrant *reg; uint8_t status = SPDK_NVME_SC_SUCCESS; bool update_sgroup = false; + struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS]; + uint32_t num_hostid = 0; int rc; rrega = cmd->cdw10 & 0x7u; @@ -1510,7 +1593,21 @@ nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns, status = SPDK_NVME_SC_RESERVATION_CONFLICT; goto exit; } + + rtype = ns->rtype; + num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list, + SPDK_NVMF_MAX_NUM_REGISTRANTS, + &ctrlr->hostid); + nvmf_ns_reservation_remove_registrant(ns, reg); + + if (!ns->rtype && num_hostid && (rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_REG_ONLY || + rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY)) { + nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns, + hostid_list, + num_hostid, + SPDK_NVME_RESERVATION_RELEASED); + } update_sgroup = true; break; case SPDK_NVME_RESERVE_REPLACE_KEY: @@ -1550,6 +1647,11 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, bool all_regs = false; uint32_t count = 0; bool update_sgroup = true; + struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS]; + uint32_t num_hostid = 0; + struct spdk_uuid new_hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS]; + uint32_t new_num_hostid = 0; + bool reservation_released = false; uint8_t status = SPDK_NVME_SC_SUCCESS; racqa = cmd->cdw10 & 0x7u; @@ -1603,18 +1705,24 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey); break; } + num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list, + SPDK_NVMF_MAX_NUM_REGISTRANTS, + &ctrlr->hostid); + /* only 1 reservation holder and reservation key is valid */ if (!all_regs) { /* preempt itself */ if (nvmf_ns_reservation_registrant_is_holder(ns, reg) && ns->crkey == key.prkey) { ns->rtype = rtype; + reservation_released = true; break; } if (ns->crkey == key.prkey) { nvmf_ns_reservation_remove_registrant(ns, ns->holder); nvmf_ns_reservation_acquire_reservation(ns, key.crkey, rtype, reg); + reservation_released = true; } else if (key.prkey != 0) { nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey); } else { @@ -1647,6 +1755,36 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, } exit: + if (update_sgroup && racqa == SPDK_NVME_RESERVE_PREEMPT) { + new_num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, new_hostid_list, + SPDK_NVMF_MAX_NUM_REGISTRANTS, + &ctrlr->hostid); + /* Preempt notification occurs on the unregistered controllers + * other than the controller who issued the command. + */ + num_hostid = nvmf_ns_reservation_get_unregistered_hostid(hostid_list, + num_hostid, + new_hostid_list, + new_num_hostid); + if (num_hostid) { + nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns, + hostid_list, + num_hostid, + SPDK_NVME_REGISTRATION_PREEMPTED); + + } + /* Reservation released notification occurs on the + * controllers which are the remaining registrants other than + * the controller who issued the command. + */ + if (reservation_released && new_num_hostid) { + nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns, + new_hostid_list, + new_num_hostid, + SPDK_NVME_RESERVATION_RELEASED); + + } + } req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; req->rsp->nvme_cpl.status.sc = status; return update_sgroup; @@ -1663,6 +1801,8 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns, uint64_t crkey; uint8_t status = SPDK_NVME_SC_SUCCESS; bool update_sgroup = true; + struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS]; + uint32_t num_hostid = 0; rrela = cmd->cdw10 & 0x7u; iekey = (cmd->cdw10 >> 3) & 0x1u; @@ -1688,6 +1828,10 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns, goto exit; } + num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list, + SPDK_NVMF_MAX_NUM_REGISTRANTS, + &ctrlr->hostid); + switch (rrela) { case SPDK_NVME_RESERVE_RELEASE: if (!ns->holder) { @@ -1706,10 +1850,26 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns, update_sgroup = false; goto exit; } + + rtype = ns->rtype; nvmf_ns_reservation_release_reservation(ns); + + if (num_hostid && rtype != SPDK_NVME_RESERVE_WRITE_EXCLUSIVE && + rtype != SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { + nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns, + hostid_list, + num_hostid, + SPDK_NVME_RESERVATION_RELEASED); + } break; case SPDK_NVME_RESERVE_CLEAR: nvmf_ns_reservation_clear_all_registrants(ns); + if (num_hostid) { + nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns, + hostid_list, + num_hostid, + SPDK_NVME_RESERVATION_PREEMPTED); + } break; default: status = SPDK_NVME_SC_INVALID_FIELD; diff --git a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c index f2bef86cb..147286d9c 100644 --- a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c +++ b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c @@ -50,6 +50,10 @@ DEFINE_STUB(spdk_bdev_module_claim_bdev, DEFINE_STUB_V(spdk_bdev_module_release_bdev, (struct spdk_bdev *bdev)); +DEFINE_STUB_V(spdk_nvmf_ctrlr_reservation_notice_log, + (struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvmf_ns *ns, + enum spdk_nvme_reservation_notification_log_page_type type)); + DEFINE_STUB(spdk_bdev_get_block_size, uint32_t, (const struct spdk_bdev *bdev), 512); @@ -458,7 +462,7 @@ test_spdk_nvmf_subsystem_set_sn(void) * | NAMESPACE 1 | * -------------------------------------- */ - +static struct spdk_nvmf_subsystem g_subsystem; static struct spdk_nvmf_ctrlr g_ctrlr1_A, g_ctrlr2_A, g_ctrlr_B, g_ctrlr_C; static struct spdk_nvmf_ns g_ns; struct spdk_nvmf_subsystem_pg_ns_info g_ns_info; @@ -466,29 +470,71 @@ struct spdk_nvmf_subsystem_pg_ns_info g_ns_info; static void ut_reservation_init(void) { + + TAILQ_INIT(&g_subsystem.ctrlrs); + memset(&g_ns, 0, sizeof(g_ns)); TAILQ_INIT(&g_ns.registrants); + g_ns.subsystem = &g_subsystem; /* Host A has two controllers */ spdk_uuid_generate(&g_ctrlr1_A.hostid); + TAILQ_INIT(&g_ctrlr1_A.log_head); + g_ctrlr1_A.subsys = &g_subsystem; + TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr1_A, link); spdk_uuid_copy(&g_ctrlr2_A.hostid, &g_ctrlr1_A.hostid); + TAILQ_INIT(&g_ctrlr2_A.log_head); + g_ctrlr2_A.subsys = &g_subsystem; + TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr2_A, link); /* Host B has 1 controller */ spdk_uuid_generate(&g_ctrlr_B.hostid); + TAILQ_INIT(&g_ctrlr_B.log_head); + g_ctrlr_B.subsys = &g_subsystem; + TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr_B, link); /* Host C has 1 controller */ spdk_uuid_generate(&g_ctrlr_C.hostid); + TAILQ_INIT(&g_ctrlr_C.log_head); + g_ctrlr_C.subsys = &g_subsystem; + TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr_C, link); } static void ut_reservation_deinit(void) { struct spdk_nvmf_registrant *reg, *tmp; + struct spdk_nvmf_reservation_log *log, *log_tmp; + struct spdk_nvmf_ctrlr *ctrlr, *ctrlr_tmp; TAILQ_FOREACH_SAFE(reg, &g_ns.registrants, link, tmp) { TAILQ_REMOVE(&g_ns.registrants, reg, link); free(reg); } + TAILQ_FOREACH_SAFE(log, &g_ctrlr1_A.log_head, link, log_tmp) { + TAILQ_REMOVE(&g_ctrlr1_A.log_head, log, link); + free(log); + } + g_ctrlr1_A.num_avail_log_pages = 0; + TAILQ_FOREACH_SAFE(log, &g_ctrlr2_A.log_head, link, log_tmp) { + TAILQ_REMOVE(&g_ctrlr2_A.log_head, log, link); + free(log); + } + g_ctrlr2_A.num_avail_log_pages = 0; + TAILQ_FOREACH_SAFE(log, &g_ctrlr_B.log_head, link, log_tmp) { + TAILQ_REMOVE(&g_ctrlr_B.log_head, log, link); + free(log); + } + g_ctrlr_B.num_avail_log_pages = 0; + TAILQ_FOREACH_SAFE(log, &g_ctrlr_C.log_head, link, log_tmp) { + TAILQ_REMOVE(&g_ctrlr_C.log_head, log, link); + free(log); + } + g_ctrlr_C.num_avail_log_pages = 0; + + TAILQ_FOREACH_SAFE(ctrlr, &g_subsystem.ctrlrs, link, ctrlr_tmp) { + TAILQ_REMOVE(&g_subsystem.ctrlrs, ctrlr, link); + } } static struct spdk_nvmf_request *