ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bdev/nvme: Reset I/O disables retry when destroying I/O qpairs

Browse Source 
As the RBD bdev module does, the upper layer wants the reset command
to abort or complete all I/Os submitted before the reset command.

To satisfy this requirement, return all aborted I/Os by deleting I/O
qpairs to the upper layer without retry. To return all aborted I/Os
by deleting I/O qpairs, enable DNR for I/O qpairs. These I/O qpairs
are deleted and recreated. Hence, we do not have to disable DNR.

No more I/O comes at a reset I/O because the generic bdev layer already
blocks I/O submission. However, some I/Os may be queued for retry even
after deleting I/O qpairs. Hence, abort all queued I/Os for the bdev
before completing the reset I/O.

Signed-off-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I9830026ef5f2b9c28aee92e6ce4018ed8541c808
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16836
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
This commit is contained in:
Shuhei Matsumoto 2023-03-08 13:39:07 +09:00 committed by David Ko
parent 8bc0582f6e
commit 610265c9fa
3 changed files with 34 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
module/bdev/nvme/bdev_nvme.c
View File

@ -1923,6 +1923,7 @@ _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status)
pthread_mutex_lock(&nvme_ctrlr->mutex);
nvme_ctrlr->resetting = false;
nvme_ctrlr->dont_retry = false;
path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
assert(path_id != NULL);
@ -1984,6 +1985,9 @@ bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i)
_bdev_nvme_clear_io_path_cache(nvme_qpair);
if (nvme_qpair->qpair != NULL) {
if (nvme_qpair->ctrlr->dont_retry) {
spdk_nvme_qpair_set_abort_dnr(nvme_qpair->qpair, true);
}
spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
/* The current full reset sequence will move to the next
@ -2133,6 +2137,7 @@ bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr)
}
nvme_ctrlr->resetting = true;
nvme_ctrlr->dont_retry = true;
if (nvme_ctrlr->reconnect_is_delayed) {
SPDK_DEBUGLOG(bdev_nvme, "Reconnect is already scheduled.\n");
@ -2167,8 +2172,9 @@ bdev_nvme_reset_rpc(struct nvme_ctrlr *nvme_ctrlr, bdev_nvme_reset_cb cb_fn, voi
static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio);
static void
bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
_bdev_nvme_reset_io_complete(struct spdk_io_channel_iter *i, int status)
{
struct nvme_bdev_io *bio = spdk_io_channel_iter_get_ctx(i);
enum spdk_bdev_io_status io_status;
if (bio->cpl.cdw0 == 0) {
@ -2180,6 +2186,30 @@ bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
__bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL);
}
static void
bdev_nvme_abort_bdev_channel(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch);
bdev_nvme_abort_retry_ios(nbdev_ch);
spdk_for_each_channel_continue(i, 0);
}
static void
bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
{
struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
/* Abort all queued I/Os for retry. */
spdk_for_each_channel(nbdev,
bdev_nvme_abort_bdev_channel,
bio,
_bdev_nvme_reset_io_complete);
}
static void
_bdev_nvme_reset_io_continue(void *ctx)
{

1
module/bdev/nvme/bdev_nvme.h
View File

@ -115,6 +115,7 @@ struct nvme_ctrlr {
uint32_t destruct : 1;
uint32_t ana_log_page_updating : 1;
uint32_t io_path_cache_clearing : 1;
uint32_t dont_retry : 1;
struct nvme_ctrlr_opts opts;

2
test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c
View File

@ -69,6 +69,8 @@ DEFINE_STUB_V(spdk_bdev_reset_io_stat, (struct spdk_bdev_io_stat *stat,
DEFINE_STUB_V(spdk_bdev_add_io_stat, (struct spdk_bdev_io_stat *total,
struct spdk_bdev_io_stat *add));
DEFINE_STUB_V(spdk_nvme_qpair_set_abort_dnr, (struct spdk_nvme_qpair *qpair, bool dnr));
int
spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size)