From 6bf92c33e5a8eedf4244c6fe391d745188ca9d4d Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Fri, 8 Dec 2017 12:32:13 -0700 Subject: [PATCH] nbd: move initialization out of kernel thread Send all of the initialization ioctls from the SPDK thread running spdk_nbd_start(), rather than in the pthread spawned to handle the blocking NBD_DO_IT ioctl. This allows the parameter to the pthread to be a single value (dev_fd), rather than a pointer to the whole spdk_nbd_disk, which fixes a race on shutdown where the main thread is freeing the spdk_nbd_disk while the kernel thread is accessing nbd->dev_fd to send the cleanup ioctls. Change-Id: Ibe5ca4abac97ac567b294da1f8a92f12eece45b8 Signed-off-by: Daniel Verkamp Reviewed-on: https://review.gerrithub.io/391021 Reviewed-by: Jim Harris Tested-by: SPDK Automated Test System Reviewed-by: Ben Walker --- lib/nbd/nbd.c | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/nbd/nbd.c b/lib/nbd/nbd.c index 45c112724..2c1e6e5fe 100644 --- a/lib/nbd/nbd.c +++ b/lib/nbd/nbd.c @@ -501,32 +501,14 @@ spdk_nbd_poll(void *arg) static void * nbd_start_kernel(void *arg) { - struct spdk_nbd_disk *nbd = arg; - int rc; - char buf[64]; + int dev_fd = (int)(intptr_t)arg; spdk_unaffinitize_thread(); - rc = ioctl(nbd->dev_fd, NBD_SET_SOCK, nbd->kernel_sp_fd); - if (rc == -1) { - spdk_strerror_r(errno, buf, sizeof(buf)); - SPDK_ERRLOG("ioctl(NBD_SET_SOCK) failed: %s\n", buf); - pthread_exit(NULL); - } - -#ifdef NBD_FLAG_SEND_TRIM - rc = ioctl(nbd->dev_fd, NBD_SET_FLAGS, NBD_FLAG_SEND_TRIM); - if (rc == -1) { - spdk_strerror_r(errno, buf, sizeof(buf)); - SPDK_ERRLOG("ioctl(NBD_SET_FLAGS) failed: %s\n", buf); - pthread_exit(NULL); - } -#endif - /* This will block in the kernel until we close the spdk_sp_fd. */ - ioctl(nbd->dev_fd, NBD_DO_IT); - ioctl(nbd->dev_fd, NBD_CLEAR_QUE); - ioctl(nbd->dev_fd, NBD_CLEAR_SOCK); + ioctl(dev_fd, NBD_DO_IT); + ioctl(dev_fd, NBD_CLEAR_QUE); + ioctl(dev_fd, NBD_CLEAR_SOCK); pthread_exit(NULL); } @@ -616,7 +598,23 @@ spdk_nbd_start(const char *bdev_name, const char *nbd_path) printf("Enabling kernel access to bdev %s via %s\n", spdk_bdev_get_name(bdev), nbd_path); - rc = pthread_create(&tid, NULL, &nbd_start_kernel, nbd); + rc = ioctl(nbd->dev_fd, NBD_SET_SOCK, nbd->kernel_sp_fd); + if (rc == -1) { + spdk_strerror_r(errno, buf, sizeof(buf)); + SPDK_ERRLOG("ioctl(NBD_SET_SOCK) failed: %s\n", buf); + goto err; + } + +#ifdef NBD_FLAG_SEND_TRIM + rc = ioctl(nbd->dev_fd, NBD_SET_FLAGS, NBD_FLAG_SEND_TRIM); + if (rc == -1) { + spdk_strerror_r(errno, buf, sizeof(buf)); + SPDK_ERRLOG("ioctl(NBD_SET_FLAGS) failed: %s\n", buf); + goto err; + } +#endif + + rc = pthread_create(&tid, NULL, nbd_start_kernel, (void *)(intptr_t)nbd->dev_fd); if (rc != 0) { spdk_strerror_r(rc, buf, sizeof(buf)); SPDK_ERRLOG("could not create thread: %s\n", buf);