ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nbd: put all fds in spdk_nbd_disk

Browse Source 
dev_fd => fd for /dev/nbdX device node
spdk_sp_fd => fd that SPDK will use to poll for new
		I/O requests from the kernel nbd thread
kernel_sp_fd => fd passed to the kernel to send I/O
		to SPDK

This enables an upcoming patch to switch from using
fork() to pthread_create().

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I0840a35eb1ba12810b7fc12775ee0df9ad3d99f9

Reviewed-on: https://review.gerrithub.io/388519
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Jim Harris 2017-11-21 12:51:57 -07:00
parent 745c0496fe
commit 1517ed376d
1 changed files with 37 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
lib/nbd/nbd.c
View File

@ -71,7 +71,9 @@ struct spdk_nbd_disk {
struct spdk_bdev *bdev; struct spdk_bdev *bdev;
struct spdk_bdev_desc *bdev_desc; struct spdk_bdev_desc *bdev_desc;
struct spdk_io_channel *ch; struct spdk_io_channel *ch;
int fd; int dev_fd;
int kernel_sp_fd;
int spdk_sp_fd;
struct nbd_io io; struct nbd_io io;
uint32_t buf_align; uint32_t buf_align;
}; };
@ -109,8 +111,16 @@ _nbd_stop(struct spdk_nbd_disk *nbd)
spdk_bdev_close(nbd->bdev_desc); spdk_bdev_close(nbd->bdev_desc);
} }
if (nbd->fd >= 0) { if (nbd->dev_fd >= 0) {
close(nbd->fd); close(nbd->dev_fd);
}
if (nbd->spdk_sp_fd >= 0) {
close(nbd->spdk_sp_fd);
}
if (nbd->kernel_sp_fd >= 0) {
close(nbd->kernel_sp_fd);
} }
free(nbd); free(nbd);
@ -275,7 +285,7 @@ int
spdk_nbd_poll(struct spdk_nbd_disk *nbd) spdk_nbd_poll(struct spdk_nbd_disk *nbd)
{ {
struct nbd_io *io = &nbd->io; struct nbd_io *io = &nbd->io;
int fd = nbd->fd; int fd = nbd->spdk_sp_fd;
int64_t ret; int64_t ret;
int rc; int rc;
@ -341,14 +351,14 @@ spdk_nbd_poll(struct spdk_nbd_disk *nbd)
} }
static void static void
nbd_start_kernel(int nbd_fd, int *sp) nbd_start_kernel(struct spdk_nbd_disk *nbd)
{ {
int rc; int rc;
char buf[64]; char buf[64];
close(sp[0]); close(nbd->spdk_sp_fd);
rc = ioctl(nbd_fd, NBD_SET_SOCK, sp[1]); rc = ioctl(nbd->dev_fd, NBD_SET_SOCK, nbd->kernel_sp_fd);
if (rc == -1) { if (rc == -1) {
spdk_strerror_r(errno, buf, sizeof(buf)); spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("ioctl(NBD_SET_SOCK) failed: %s\n", buf); SPDK_ERRLOG("ioctl(NBD_SET_SOCK) failed: %s\n", buf);
@ -356,7 +366,7 @@ nbd_start_kernel(int nbd_fd, int *sp)
} }
#ifdef NBD_FLAG_SEND_TRIM #ifdef NBD_FLAG_SEND_TRIM
rc = ioctl(nbd_fd, NBD_SET_FLAGS, NBD_FLAG_SEND_TRIM); rc = ioctl(nbd->dev_fd, NBD_SET_FLAGS, NBD_FLAG_SEND_TRIM);
if (rc == -1) { if (rc == -1) {
spdk_strerror_r(errno, buf, sizeof(buf)); spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("ioctl(NBD_SET_FLAGS) failed: %s\n", buf); SPDK_ERRLOG("ioctl(NBD_SET_FLAGS) failed: %s\n", buf);
@ -365,10 +375,10 @@ nbd_start_kernel(int nbd_fd, int *sp)
#endif #endif
/* This will block in the kernel until the client disconnects. */ /* This will block in the kernel until the client disconnects. */
ioctl(nbd_fd, NBD_DO_IT); ioctl(nbd->dev_fd, NBD_DO_IT);
ioctl(nbd_fd, NBD_CLEAR_QUE); ioctl(nbd->dev_fd, NBD_CLEAR_QUE);
ioctl(nbd_fd, NBD_CLEAR_SOCK); ioctl(nbd->dev_fd, NBD_CLEAR_SOCK);
exit(0); exit(0);
} }
@ -378,14 +388,16 @@ spdk_nbd_start(struct spdk_bdev *bdev, const char *nbd_path)
{ {
struct spdk_nbd_disk *nbd; struct spdk_nbd_disk *nbd;
int rc; int rc;
int sp[2] = { -1, -1 }, nbd_fd = -1; int sp[2];
char buf[64]; char buf[64];
nbd = calloc(1, sizeof(*nbd)); nbd = calloc(1, sizeof(*nbd));
if (nbd == NULL) { if (nbd == NULL) {
return NULL; return NULL;
} }
nbd->fd = -1; nbd->dev_fd = -1;
nbd->spdk_sp_fd = -1;
nbd->kernel_sp_fd = -1;
rc = spdk_bdev_open(bdev, true, NULL, NULL, &nbd->bdev_desc); rc = spdk_bdev_open(bdev, true, NULL, NULL, &nbd->bdev_desc);
if (rc != 0) { if (rc != 0) {
@ -404,28 +416,30 @@ spdk_nbd_start(struct spdk_bdev *bdev, const char *nbd_path)
goto err; goto err;
} }
nbd_fd = open(nbd_path, O_RDWR); nbd->spdk_sp_fd = sp[0];
if (nbd_fd == -1) { nbd->kernel_sp_fd = sp[1];
nbd->dev_fd = open(nbd_path, O_RDWR);
if (nbd->dev_fd == -1) {
spdk_strerror_r(errno, buf, sizeof(buf)); spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("open(\"%s\") failed: %s\n", nbd_path, buf); SPDK_ERRLOG("open(\"%s\") failed: %s\n", nbd_path, buf);
goto err; goto err;
} }
rc = ioctl(nbd_fd, NBD_SET_BLKSIZE, spdk_bdev_get_block_size(bdev)); rc = ioctl(nbd->dev_fd, NBD_SET_BLKSIZE, spdk_bdev_get_block_size(bdev));
if (rc == -1) { if (rc == -1) {
spdk_strerror_r(errno, buf, sizeof(buf)); spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("ioctl(NBD_SET_BLKSIZE) failed: %s\n", buf); SPDK_ERRLOG("ioctl(NBD_SET_BLKSIZE) failed: %s\n", buf);
goto err; goto err;
} }
rc = ioctl(nbd_fd, NBD_SET_SIZE_BLOCKS, spdk_bdev_get_num_blocks(bdev)); rc = ioctl(nbd->dev_fd, NBD_SET_SIZE_BLOCKS, spdk_bdev_get_num_blocks(bdev));
if (rc == -1) { if (rc == -1) {
spdk_strerror_r(errno, buf, sizeof(buf)); spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("ioctl(NBD_SET_SIZE_BLOCKS) failed: %s\n", buf); SPDK_ERRLOG("ioctl(NBD_SET_SIZE_BLOCKS) failed: %s\n", buf);
goto err; goto err;
} }
rc = ioctl(nbd_fd, NBD_CLEAR_SOCK); rc = ioctl(nbd->dev_fd, NBD_CLEAR_SOCK);
if (rc == -1) { if (rc == -1) {
spdk_strerror_r(errno, buf, sizeof(buf)); spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("ioctl(NBD_CLEAR_SOCK) failed: %s\n", buf); SPDK_ERRLOG("ioctl(NBD_CLEAR_SOCK) failed: %s\n", buf);
@ -438,21 +452,20 @@ spdk_nbd_start(struct spdk_bdev *bdev, const char *nbd_path)
switch (rc) { switch (rc) {
case 0: case 0:
nbd_start_kernel(nbd_fd, sp); nbd_start_kernel(nbd);
break; break;
case -1: case -1:
spdk_strerror_r(errno, buf, sizeof(buf)); spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("could not fork: %s\n", buf); SPDK_ERRLOG("could not fork: %s\n", buf);
goto err; goto err;
default: default:
close(nbd_fd); close(nbd->dev_fd);
break; break;
} }
close(sp[1]); close(nbd->kernel_sp_fd);
nbd->fd = sp[0]; fcntl(nbd->spdk_sp_fd, F_SETFL, O_NONBLOCK);
fcntl(nbd->fd, F_SETFL, O_NONBLOCK);
to_be32(&nbd->io.resp.magic, NBD_REPLY_MAGIC); to_be32(&nbd->io.resp.magic, NBD_REPLY_MAGIC);
nbd->io.req_in_progress = true; nbd->io.req_in_progress = true;
@ -460,18 +473,6 @@ spdk_nbd_start(struct spdk_bdev *bdev, const char *nbd_path)
return nbd; return nbd;
err: err:
if (sp[0] >= 0) {
close(sp[0]);
}
if (sp[1] >= 0) {
close(sp[1]);
}
if (nbd_fd >= 0) {
close(nbd_fd);
}
spdk_nbd_stop(nbd); spdk_nbd_stop(nbd);
return NULL; return NULL;