nvmf: generate reservation notification log pages

A host can use the Asynchronous Event Command to be notified of
the presense of one or more avaiable reservation notification
log pages.  A reservation notificaton log page should be created
whenever an unmasked reservation notification occurs.

Change-Id: I8b83e5319725286dd0a5efc1b22d8ac4673e31e1
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/439931
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Changpeng Liu 2019-03-01 03:58:57 -05:00
parent 8cc7b0bc4d
commit 78bfb2a1d0
4 changed files with 280 additions and 2 deletions

View File

@ -269,6 +269,7 @@ spdk_nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem,
return NULL;
}
TAILQ_INIT(&ctrlr->log_head);
ctrlr->subsys = subsystem;
ctrlr->thread = req->qpair->group->thread;
@ -361,8 +362,14 @@ static void
_spdk_nvmf_ctrlr_destruct(void *ctx)
{
struct spdk_nvmf_ctrlr *ctrlr = ctx;
struct spdk_nvmf_reservation_log *log, *log_tmp;
spdk_nvmf_ctrlr_stop_keep_alive_timer(ctrlr);
TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) {
TAILQ_REMOVE(&ctrlr->log_head, log, link);
free(log);
}
free(ctrlr);
}
@ -2080,6 +2087,57 @@ spdk_nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr)
ctrlr->aer_req = NULL;
}
void
spdk_nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr,
struct spdk_nvmf_ns *ns,
enum spdk_nvme_reservation_notification_log_page_type type)
{
struct spdk_nvmf_reservation_log *log;
switch (type) {
case SPDK_NVME_RESERVATION_LOG_PAGE_EMPTY:
return;
case SPDK_NVME_REGISTRATION_PREEMPTED:
if (ns->mask & SPDK_NVME_REGISTRATION_PREEMPTED_MASK) {
return;
}
break;
case SPDK_NVME_RESERVATION_RELEASED:
if (ns->mask & SPDK_NVME_RESERVATION_RELEASED_MASK) {
return;
}
break;
case SPDK_NVME_RESERVATION_PREEMPTED:
if (ns->mask & SPDK_NVME_RESERVATION_PREEMPTED_MASK) {
return;
}
break;
default:
return;
}
ctrlr->log_page_count++;
/* Maximum number of queued log pages is 255 */
if (ctrlr->num_avail_log_pages == 0xff) {
log = TAILQ_LAST(&ctrlr->log_head, log_page_head);
log->log.log_page_count = ctrlr->log_page_count;
return;
}
log = calloc(1, sizeof(*log));
if (!log) {
SPDK_ERRLOG("Alloc log page failed, ignore the log\n");
return;
}
log->log.log_page_count = ctrlr->log_page_count;
log->log.type = type;
log->log.num_avail_log_pages = ctrlr->num_avail_log_pages++;
log->log.nsid = ns->nsid;
TAILQ_INSERT_TAIL(&ctrlr->log_head, log, link);
}
/* Check from subsystem poll group's namespace information data structure */
static bool
nvmf_ns_info_ctrlr_is_registrant(struct spdk_nvmf_subsystem_pg_ns_info *ns_info,

View File

@ -241,6 +241,14 @@ struct spdk_nvmf_ctrlr_feat {
union spdk_nvme_feat_keep_alive_timer keep_alive_timer;
};
/*
* NVMf reservation notificaton log page.
*/
struct spdk_nvmf_reservation_log {
struct spdk_nvme_reservation_notification_log log;
TAILQ_ENTRY(spdk_nvmf_reservation_log) link;
};
/*
* This structure represents an NVMe-oF controller,
* which is like a "session" in networking terms.
@ -268,6 +276,9 @@ struct spdk_nvmf_ctrlr {
uint16_t changed_ns_list_count;
struct spdk_nvme_ns_list changed_ns_list;
uint64_t log_page_count;
uint8_t num_avail_log_pages;
TAILQ_HEAD(log_page_head, spdk_nvmf_reservation_log) log_head;
/* Time to trigger keep-alive--poller_time = now_tick + period */
uint64_t last_keep_alive_tick;
@ -360,6 +371,9 @@ struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem
uint16_t cntlid);
int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr);
void spdk_nvmf_ns_reservation_request(void *ctx);
void spdk_nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr,
struct spdk_nvmf_ns *ns,
enum spdk_nvme_reservation_notification_log_page_type type);
/*
* Abort aer is sent on a per controller basis and sends a completion for the aer to the host.

View File

@ -1303,6 +1303,87 @@ nvmf_ns_reservation_get_registrant(struct spdk_nvmf_ns *ns,
return NULL;
}
/* Generate reservation notice log to registered HostID controllers */
static void
nvmf_subsystem_gen_ctrlr_notification(struct spdk_nvmf_subsystem *subsystem,
struct spdk_nvmf_ns *ns,
struct spdk_uuid *hostid_list,
uint32_t num_hostid,
enum spdk_nvme_reservation_notification_log_page_type type)
{
struct spdk_nvmf_ctrlr *ctrlr;
uint32_t i;
for (i = 0; i < num_hostid; i++) {
TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) {
if (!spdk_uuid_compare(&ctrlr->hostid, &hostid_list[i])) {
spdk_nvmf_ctrlr_reservation_notice_log(ctrlr, ns, type);
}
}
}
}
/* Get all registrants' hostid other than the controller who issued the command */
static uint32_t
nvmf_ns_reservation_get_all_other_hostid(struct spdk_nvmf_ns *ns,
struct spdk_uuid *hostid_list,
uint32_t max_num_hostid,
struct spdk_uuid *current_hostid)
{
struct spdk_nvmf_registrant *reg, *tmp;
uint32_t num_hostid = 0;
TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) {
if (spdk_uuid_compare(&reg->hostid, current_hostid)) {
if (num_hostid == max_num_hostid) {
assert(false);
return max_num_hostid;
}
hostid_list[num_hostid++] = reg->hostid;
}
}
return num_hostid;
}
/* Calculate the unregistered HostID list according to list
* prior to execute preempt command and list after executing
* preempt command.
*/
static uint32_t
nvmf_ns_reservation_get_unregistered_hostid(struct spdk_uuid *old_hostid_list,
uint32_t old_num_hostid,
struct spdk_uuid *remaining_hostid_list,
uint32_t remaining_num_hostid)
{
struct spdk_uuid temp_hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];
uint32_t i, j, num_hostid = 0;
bool found;
if (!remaining_num_hostid) {
return old_num_hostid;
}
for (i = 0; i < old_num_hostid; i++) {
found = false;
for (j = 0; j < remaining_num_hostid; j++) {
if (!spdk_uuid_compare(&old_hostid_list[i], &remaining_hostid_list[j])) {
found = true;
break;
}
}
if (!found) {
spdk_uuid_copy(&temp_hostid_list[num_hostid++], &old_hostid_list[i]);
}
}
if (num_hostid) {
memcpy(old_hostid_list, temp_hostid_list, sizeof(struct spdk_uuid) * num_hostid);
}
return num_hostid;
}
/* current reservation type is all registrants or not */
static bool
nvmf_ns_reservation_all_registrants_type(struct spdk_nvmf_ns *ns)
@ -1452,11 +1533,13 @@ nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns,
struct spdk_nvmf_request *req)
{
struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
uint8_t rrega, iekey, cptpl;
uint8_t rrega, iekey, cptpl, rtype;
struct spdk_nvme_reservation_register_data key;
struct spdk_nvmf_registrant *reg;
uint8_t status = SPDK_NVME_SC_SUCCESS;
bool update_sgroup = false;
struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];
uint32_t num_hostid = 0;
int rc;
rrega = cmd->cdw10 & 0x7u;
@ -1510,7 +1593,21 @@ nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns,
status = SPDK_NVME_SC_RESERVATION_CONFLICT;
goto exit;
}
rtype = ns->rtype;
num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,
SPDK_NVMF_MAX_NUM_REGISTRANTS,
&ctrlr->hostid);
nvmf_ns_reservation_remove_registrant(ns, reg);
if (!ns->rtype && num_hostid && (rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_REG_ONLY ||
rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY)) {
nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,
hostid_list,
num_hostid,
SPDK_NVME_RESERVATION_RELEASED);
}
update_sgroup = true;
break;
case SPDK_NVME_RESERVE_REPLACE_KEY:
@ -1550,6 +1647,11 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns,
bool all_regs = false;
uint32_t count = 0;
bool update_sgroup = true;
struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];
uint32_t num_hostid = 0;
struct spdk_uuid new_hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];
uint32_t new_num_hostid = 0;
bool reservation_released = false;
uint8_t status = SPDK_NVME_SC_SUCCESS;
racqa = cmd->cdw10 & 0x7u;
@ -1603,18 +1705,24 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns,
nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey);
break;
}
num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,
SPDK_NVMF_MAX_NUM_REGISTRANTS,
&ctrlr->hostid);
/* only 1 reservation holder and reservation key is valid */
if (!all_regs) {
/* preempt itself */
if (nvmf_ns_reservation_registrant_is_holder(ns, reg) &&
ns->crkey == key.prkey) {
ns->rtype = rtype;
reservation_released = true;
break;
}
if (ns->crkey == key.prkey) {
nvmf_ns_reservation_remove_registrant(ns, ns->holder);
nvmf_ns_reservation_acquire_reservation(ns, key.crkey, rtype, reg);
reservation_released = true;
} else if (key.prkey != 0) {
nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey);
} else {
@ -1647,6 +1755,36 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns,
}
exit:
if (update_sgroup && racqa == SPDK_NVME_RESERVE_PREEMPT) {
new_num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, new_hostid_list,
SPDK_NVMF_MAX_NUM_REGISTRANTS,
&ctrlr->hostid);
/* Preempt notification occurs on the unregistered controllers
* other than the controller who issued the command.
*/
num_hostid = nvmf_ns_reservation_get_unregistered_hostid(hostid_list,
num_hostid,
new_hostid_list,
new_num_hostid);
if (num_hostid) {
nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,
hostid_list,
num_hostid,
SPDK_NVME_REGISTRATION_PREEMPTED);
}
/* Reservation released notification occurs on the
* controllers which are the remaining registrants other than
* the controller who issued the command.
*/
if (reservation_released && new_num_hostid) {
nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,
new_hostid_list,
new_num_hostid,
SPDK_NVME_RESERVATION_RELEASED);
}
}
req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
req->rsp->nvme_cpl.status.sc = status;
return update_sgroup;
@ -1663,6 +1801,8 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns,
uint64_t crkey;
uint8_t status = SPDK_NVME_SC_SUCCESS;
bool update_sgroup = true;
struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];
uint32_t num_hostid = 0;
rrela = cmd->cdw10 & 0x7u;
iekey = (cmd->cdw10 >> 3) & 0x1u;
@ -1688,6 +1828,10 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns,
goto exit;
}
num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,
SPDK_NVMF_MAX_NUM_REGISTRANTS,
&ctrlr->hostid);
switch (rrela) {
case SPDK_NVME_RESERVE_RELEASE:
if (!ns->holder) {
@ -1706,10 +1850,26 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns,
update_sgroup = false;
goto exit;
}
rtype = ns->rtype;
nvmf_ns_reservation_release_reservation(ns);
if (num_hostid && rtype != SPDK_NVME_RESERVE_WRITE_EXCLUSIVE &&
rtype != SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) {
nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,
hostid_list,
num_hostid,
SPDK_NVME_RESERVATION_RELEASED);
}
break;
case SPDK_NVME_RESERVE_CLEAR:
nvmf_ns_reservation_clear_all_registrants(ns);
if (num_hostid) {
nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,
hostid_list,
num_hostid,
SPDK_NVME_RESERVATION_PREEMPTED);
}
break;
default:
status = SPDK_NVME_SC_INVALID_FIELD;

View File

@ -50,6 +50,10 @@ DEFINE_STUB(spdk_bdev_module_claim_bdev,
DEFINE_STUB_V(spdk_bdev_module_release_bdev,
(struct spdk_bdev *bdev));
DEFINE_STUB_V(spdk_nvmf_ctrlr_reservation_notice_log,
(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvmf_ns *ns,
enum spdk_nvme_reservation_notification_log_page_type type));
DEFINE_STUB(spdk_bdev_get_block_size, uint32_t,
(const struct spdk_bdev *bdev), 512);
@ -458,7 +462,7 @@ test_spdk_nvmf_subsystem_set_sn(void)
* | NAMESPACE 1 |
* --------------------------------------
*/
static struct spdk_nvmf_subsystem g_subsystem;
static struct spdk_nvmf_ctrlr g_ctrlr1_A, g_ctrlr2_A, g_ctrlr_B, g_ctrlr_C;
static struct spdk_nvmf_ns g_ns;
struct spdk_nvmf_subsystem_pg_ns_info g_ns_info;
@ -466,29 +470,71 @@ struct spdk_nvmf_subsystem_pg_ns_info g_ns_info;
static void
ut_reservation_init(void)
{
TAILQ_INIT(&g_subsystem.ctrlrs);
memset(&g_ns, 0, sizeof(g_ns));
TAILQ_INIT(&g_ns.registrants);
g_ns.subsystem = &g_subsystem;
/* Host A has two controllers */
spdk_uuid_generate(&g_ctrlr1_A.hostid);
TAILQ_INIT(&g_ctrlr1_A.log_head);
g_ctrlr1_A.subsys = &g_subsystem;
TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr1_A, link);
spdk_uuid_copy(&g_ctrlr2_A.hostid, &g_ctrlr1_A.hostid);
TAILQ_INIT(&g_ctrlr2_A.log_head);
g_ctrlr2_A.subsys = &g_subsystem;
TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr2_A, link);
/* Host B has 1 controller */
spdk_uuid_generate(&g_ctrlr_B.hostid);
TAILQ_INIT(&g_ctrlr_B.log_head);
g_ctrlr_B.subsys = &g_subsystem;
TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr_B, link);
/* Host C has 1 controller */
spdk_uuid_generate(&g_ctrlr_C.hostid);
TAILQ_INIT(&g_ctrlr_C.log_head);
g_ctrlr_C.subsys = &g_subsystem;
TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr_C, link);
}
static void
ut_reservation_deinit(void)
{
struct spdk_nvmf_registrant *reg, *tmp;
struct spdk_nvmf_reservation_log *log, *log_tmp;
struct spdk_nvmf_ctrlr *ctrlr, *ctrlr_tmp;
TAILQ_FOREACH_SAFE(reg, &g_ns.registrants, link, tmp) {
TAILQ_REMOVE(&g_ns.registrants, reg, link);
free(reg);
}
TAILQ_FOREACH_SAFE(log, &g_ctrlr1_A.log_head, link, log_tmp) {
TAILQ_REMOVE(&g_ctrlr1_A.log_head, log, link);
free(log);
}
g_ctrlr1_A.num_avail_log_pages = 0;
TAILQ_FOREACH_SAFE(log, &g_ctrlr2_A.log_head, link, log_tmp) {
TAILQ_REMOVE(&g_ctrlr2_A.log_head, log, link);
free(log);
}
g_ctrlr2_A.num_avail_log_pages = 0;
TAILQ_FOREACH_SAFE(log, &g_ctrlr_B.log_head, link, log_tmp) {
TAILQ_REMOVE(&g_ctrlr_B.log_head, log, link);
free(log);
}
g_ctrlr_B.num_avail_log_pages = 0;
TAILQ_FOREACH_SAFE(log, &g_ctrlr_C.log_head, link, log_tmp) {
TAILQ_REMOVE(&g_ctrlr_C.log_head, log, link);
free(log);
}
g_ctrlr_C.num_avail_log_pages = 0;
TAILQ_FOREACH_SAFE(ctrlr, &g_subsystem.ctrlrs, link, ctrlr_tmp) {
TAILQ_REMOVE(&g_subsystem.ctrlrs, ctrlr, link);
}
}
static struct spdk_nvmf_request *