ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nbd: get nbd_stop procedure in async

Browse Source 
It is possible that nbd pthread is created but not executed,
then spdk_nbd_stop is call before nbd_pthread's execution,
but nbd pthread starts to execute while nbd is totally stopped.

This patch can get spdk_stop_nbd aligned with nbd pthread.

Change-Id: I57cc92b94d36cd706616c9058134f716f0812892
Signed-off-by: Liu Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6278
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: <dongx.yi@intel.com>
This commit is contained in:
Liu Xiaodong 2021-02-05 11:24:01 -05:00 committed by Tomasz Zawadzki
parent 4c6c794be3
commit 22412af1d2
1 changed files with 40 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
lib/nbd/nbd.c
View File

@ -50,6 +50,7 @@
#define GET_IO_LOOP_COUNT 16 #define GET_IO_LOOP_COUNT 16
#define NBD_START_BUSY_WAITING_MS 1000 #define NBD_START_BUSY_WAITING_MS 1000
#define NBD_STOP_BUSY_WAITING_MS 10000
#define NBD_BUSY_POLLING_INTERVAL_US 20000 #define NBD_BUSY_POLLING_INTERVAL_US 20000
#define NBD_IO_TIMEOUT_S 60 #define NBD_IO_TIMEOUT_S 60
@ -108,6 +109,8 @@ struct spdk_nbd_disk {
struct spdk_poller *retry_poller; struct spdk_poller *retry_poller;
int retry_count; int retry_count;
/* Synchronize nbd_start_kernel pthread and nbd_stop */
bool has_nbd_pthread;
struct nbd_io *io_in_recv; struct nbd_io *io_in_recv;
TAILQ_HEAD(, nbd_io) received_io_list; TAILQ_HEAD(, nbd_io) received_io_list;
@ -370,9 +373,11 @@ nbd_cleanup_io(struct spdk_nbd_disk *nbd)
return 0; return 0;
} }
static void static int
_nbd_stop(struct spdk_nbd_disk *nbd) _nbd_stop(void *arg)
{ {
struct spdk_nbd_disk *nbd = arg;
if (nbd->nbd_poller) { if (nbd->nbd_poller) {
spdk_poller_unregister(&nbd->nbd_poller); spdk_poller_unregister(&nbd->nbd_poller);
} }
@ -383,10 +388,32 @@ _nbd_stop(struct spdk_nbd_disk *nbd)
if (nbd->spdk_sp_fd >= 0) { if (nbd->spdk_sp_fd >= 0) {
close(nbd->spdk_sp_fd); close(nbd->spdk_sp_fd);
nbd->spdk_sp_fd = -1;
} }
if (nbd->kernel_sp_fd >= 0) { if (nbd->kernel_sp_fd >= 0) {
close(nbd->kernel_sp_fd); close(nbd->kernel_sp_fd);
nbd->kernel_sp_fd = -1;
}
/* Continue the stop procedure after the exit of nbd_start_kernel pthread */
if (nbd->has_nbd_pthread) {
if (nbd->retry_poller == NULL) {
nbd->retry_count = NBD_STOP_BUSY_WAITING_MS * 1000ULL / NBD_BUSY_POLLING_INTERVAL_US;
nbd->retry_poller = SPDK_POLLER_REGISTER(_nbd_stop, nbd,
NBD_BUSY_POLLING_INTERVAL_US);
return SPDK_POLLER_BUSY;
}
if (nbd->retry_count-- > 0) {
return SPDK_POLLER_BUSY;
}
SPDK_ERRLOG("Failed to wait for returning of NBD_DO_IT ioctl.\n");
}
if (nbd->retry_poller) {
spdk_poller_unregister(&nbd->retry_poller);
} }
if (nbd->dev_fd >= 0) { if (nbd->dev_fd >= 0) {
@ -404,15 +431,19 @@ _nbd_stop(struct spdk_nbd_disk *nbd)
if (nbd->ch) { if (nbd->ch) {
spdk_put_io_channel(nbd->ch); spdk_put_io_channel(nbd->ch);
nbd->ch = NULL;
} }
if (nbd->bdev_desc) { if (nbd->bdev_desc) {
spdk_bdev_close(nbd->bdev_desc); spdk_bdev_close(nbd->bdev_desc);
nbd->bdev_desc = NULL;
} }
nbd_disk_unregister(nbd); nbd_disk_unregister(nbd);
free(nbd); free(nbd);
return 0;
} }
int int
@ -876,12 +907,14 @@ nbd_poll(void *arg)
static void * static void *
nbd_start_kernel(void *arg) nbd_start_kernel(void *arg)
{ {
int dev_fd = (int)(intptr_t)arg; struct spdk_nbd_disk *nbd = arg;
spdk_unaffinitize_thread(); spdk_unaffinitize_thread();
/* This will block in the kernel until we close the spdk_sp_fd. */ /* This will block in the kernel until we close the spdk_sp_fd. */
ioctl(dev_fd, NBD_DO_IT); ioctl(nbd->dev_fd, NBD_DO_IT);
nbd->has_nbd_pthread = false;
pthread_exit(NULL); pthread_exit(NULL);
} }
@ -960,8 +993,10 @@ nbd_start_complete(struct spdk_nbd_start_ctx *ctx)
} }
} }
rc = pthread_create(&tid, NULL, nbd_start_kernel, (void *)(intptr_t)ctx->nbd->dev_fd); ctx->nbd->has_nbd_pthread = true;
rc = pthread_create(&tid, NULL, nbd_start_kernel, ctx->nbd);
if (rc != 0) { if (rc != 0) {
ctx->nbd->has_nbd_pthread = false;
SPDK_ERRLOG("could not create thread: %s\n", spdk_strerror(rc)); SPDK_ERRLOG("could not create thread: %s\n", spdk_strerror(rc));
rc = -rc; rc = -rc;
goto err; goto err;