ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nbd: refactor nbd io workflow

Browse Source 
Apply a state for nbd io instead of using flag
bits like payload_in_progress and resp_in_progress.

Change-Id: I331567f7c4d7a6a886a145390a8b1e4db0f6040c
Signed-off-by: Xiaodong Liu <xiaodong.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/392175
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Xiaodong Liu 2017-12-18 01:39:15 -05:00 committed by Jim Harris
parent b5f4a259bd
commit 90ef3021bc
1 changed files with 196 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

354
lib/nbd/nbd.c
View File

@ -47,21 +47,31 @@
#include "spdk_internal/log.h" #include "spdk_internal/log.h"
enum nbd_io_state_t {
/* Receiving or ready to receive nbd request header */
NBD_IO_RECV_REQ = 0,
/* Receiving write payload */
NBD_IO_RECV_PAYLOAD,
/* processing or ready to process of SPDK bdev module */
NBD_IO_PROCESS,
/* Transmitting or ready to transmit nbd response header */
NBD_IO_XMIT_RESP,
/* Transmitting read payload */
NBD_IO_XMIT_PAYLOAD,
};
struct nbd_io { struct nbd_io {
enum spdk_bdev_io_type type; struct spdk_nbd_disk *nbd;
enum nbd_io_state_t state;
int ref; int ref;
void *payload; void *payload;
/* NOTE: for TRIM, this represents number of bytes to trim. */ /* NOTE: for TRIM, this represents number of bytes to trim. */
uint32_t payload_size; uint32_t payload_size;
bool payload_in_progress;
struct nbd_request req; struct nbd_request req;
bool req_in_progress;
struct nbd_reply resp; struct nbd_reply resp;
bool resp_in_progress;
/* /*
* Tracks current progress on reading/writing a request, * Tracks current progress on reading/writing a request,
@ -184,28 +194,6 @@ spdk_nbd_disk_get_bdev_name(struct spdk_nbd_disk *nbd)
return spdk_bdev_get_name(nbd->bdev); return spdk_bdev_get_name(nbd->bdev);
} }
static bool
is_read(enum spdk_bdev_io_type io_type)
{
if (io_type == SPDK_BDEV_IO_TYPE_READ) {
return true;
} else {
return false;
}
}
static bool
is_write(enum spdk_bdev_io_type io_type)
{
switch (io_type) {
case SPDK_BDEV_IO_TYPE_WRITE:
case SPDK_BDEV_IO_TYPE_UNMAP:
return true;
default:
return false;
}
}
void void
nbd_disconnect(struct spdk_nbd_disk *nbd) nbd_disconnect(struct spdk_nbd_disk *nbd)
{ {
@ -308,108 +296,210 @@ static void
nbd_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) nbd_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{ {
struct nbd_io *io = cb_arg; struct nbd_io *io = cb_arg;
struct spdk_nbd_disk *nbd = io->nbd;
if (success) { if (success) {
io->resp.error = 0; io->resp.error = 0;
} else { } else {
to_be32(&io->resp.error, EIO); to_be32(&io->resp.error, EIO);
} }
io->resp_in_progress = true;
memcpy(&io->resp.handle, &io->req.handle, sizeof(io->resp.handle));
io->state = NBD_IO_XMIT_RESP;
if (bdev_io != NULL) { if (bdev_io != NULL) {
spdk_bdev_free_io(bdev_io); spdk_bdev_free_io(bdev_io);
} }
io->ref--; io->ref--;
if (io->ref == 0) { if (io->ref == 0) {
_nbd_stop(SPDK_CONTAINEROF(io, struct spdk_nbd_disk, io)); _nbd_stop(nbd);
}
}
static void
nbd_submit_bdev_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
struct spdk_io_channel *ch, struct nbd_io *io)
{
int rc;
io->ref++;
switch (io->type) {
case SPDK_BDEV_IO_TYPE_READ:
rc = spdk_bdev_read(desc, ch, io->payload, from_be64(&io->req.from),
io->payload_size, nbd_io_done, io);
break;
case SPDK_BDEV_IO_TYPE_WRITE:
rc = spdk_bdev_write(desc, ch, io->payload, from_be64(&io->req.from),
io->payload_size, nbd_io_done, io);
break;
case SPDK_BDEV_IO_TYPE_UNMAP:
rc = spdk_bdev_unmap(desc, ch, from_be64(&io->req.from),
io->payload_size, nbd_io_done, io);
break;
case SPDK_BDEV_IO_TYPE_FLUSH:
rc = spdk_bdev_flush(desc, ch, 0, spdk_bdev_get_num_blocks(bdev) * spdk_bdev_get_block_size(bdev),
nbd_io_done, io);
break;
default:
rc = -1;
break;
}
if (rc == -1) {
nbd_io_done(NULL, false, io);
} }
} }
static int static int
process_request(struct spdk_nbd_disk *nbd) nbd_submit_bdev_io(struct spdk_nbd_disk *nbd, struct nbd_io *io)
{ {
struct nbd_io *io = &nbd->io; struct spdk_bdev_desc *desc = nbd->bdev_desc;
struct spdk_io_channel *ch = nbd->ch;
int rc = 0;
memcpy(&io->resp.handle, &io->req.handle, sizeof(io->resp.handle)); io->ref++;
io->resp.error = 0;
io->offset = 0;
io->payload_size = from_be32(&io->req.len);
spdk_dma_free(io->payload);
if (io->payload_size) {
io->payload = spdk_dma_malloc(io->payload_size, nbd->buf_align, NULL);
if (io->payload == NULL) {
SPDK_ERRLOG("could not allocate io->payload of size %d\n", io->payload_size);
return -ENOMEM;
}
} else {
io->payload = NULL;
}
if (from_be32(&io->req.magic) != NBD_REQUEST_MAGIC) {
SPDK_ERRLOG("invalid request magic\n");
return -EINVAL;
}
switch (from_be32(&io->req.type)) { switch (from_be32(&io->req.type)) {
case NBD_CMD_READ: case NBD_CMD_READ:
io->type = SPDK_BDEV_IO_TYPE_READ; rc = spdk_bdev_read(desc, ch, io->payload, from_be64(&io->req.from),
nbd_submit_bdev_io(nbd->bdev, nbd->bdev_desc, nbd->ch, io); io->payload_size, nbd_io_done, io);
break; break;
case NBD_CMD_WRITE: case NBD_CMD_WRITE:
io->type = SPDK_BDEV_IO_TYPE_WRITE; rc = spdk_bdev_write(desc, ch, io->payload, from_be64(&io->req.from),
io->payload_in_progress = true; io->payload_size, nbd_io_done, io);
break; break;
case NBD_CMD_DISC:
return -ECONNRESET;
#ifdef NBD_FLAG_SEND_FLUSH #ifdef NBD_FLAG_SEND_FLUSH
case NBD_CMD_FLUSH: case NBD_CMD_FLUSH:
io->type = SPDK_BDEV_IO_TYPE_FLUSH; rc = spdk_bdev_flush(desc, ch, 0,
nbd_submit_bdev_io(nbd->bdev, nbd->bdev_desc, nbd->ch, io); spdk_bdev_get_num_blocks(nbd->bdev) * spdk_bdev_get_block_size(nbd->bdev),
nbd_io_done, io);
break; break;
#endif #endif
#ifdef NBD_FLAG_SEND_TRIM #ifdef NBD_FLAG_SEND_TRIM
case NBD_CMD_TRIM: case NBD_CMD_TRIM:
io->type = SPDK_BDEV_IO_TYPE_UNMAP; rc = spdk_bdev_unmap(desc, ch, from_be64(&io->req.from),
nbd_submit_bdev_io(nbd->bdev, nbd->bdev_desc, nbd->ch, io); io->payload_size, nbd_io_done, io);
break; break;
#endif #endif
case NBD_CMD_DISC:
io->ref--;
return -ECONNRESET;
default:
rc = -1;
}
if (rc < 0) {
nbd_io_done(NULL, false, io);
}
return 0;
}
static int
spdk_nbd_io_exec(struct nbd_io *io)
{
struct spdk_nbd_disk *nbd = io->nbd;
int ret = 0;
if (io->state == NBD_IO_PROCESS) {
ret = nbd_submit_bdev_io(nbd, io);
}
return ret;
}
static int
spdk_nbd_io_recv(struct nbd_io *io)
{
struct spdk_nbd_disk *nbd = io->nbd;
int ret = 0;
if (io->state == NBD_IO_RECV_REQ) {
ret = read_from_socket(nbd->spdk_sp_fd, (char *)&io->req + io->offset,
sizeof(io->req) - io->offset);
if (ret <= 0) {
return ret;
}
io->offset += ret;
/* request is fully received */
if (io->offset == sizeof(io->req)) {
io->offset = 0;
/* req magic check */
if (from_be32(&io->req.magic) != NBD_REQUEST_MAGIC) {
SPDK_ERRLOG("invalid request magic\n");
return -EINVAL;
}
/* io except read/write should ignore payload */
if (from_be32(&io->req.type) == NBD_CMD_WRITE ||
from_be32(&io->req.type) == NBD_CMD_READ) {
io->payload_size = from_be32(&io->req.len);
} else {
io->payload_size = 0;
}
/* io payload allocate */
if (io->payload_size) {
io->payload = spdk_dma_malloc(io->payload_size, nbd->buf_align, NULL);
if (io->payload == NULL) {
SPDK_ERRLOG("could not allocate io->payload of size %d\n", io->payload_size);
return -ENOMEM;
}
} else {
io->payload = NULL;
}
/* next io step */
if (from_be32(&io->req.type) == NBD_CMD_WRITE) {
io->state = NBD_IO_RECV_PAYLOAD;
} else {
io->state = NBD_IO_PROCESS;
ret = spdk_nbd_io_exec(io);
}
}
}
if (io->state == NBD_IO_RECV_PAYLOAD) {
ret = read_from_socket(nbd->spdk_sp_fd, io->payload + io->offset, io->payload_size - io->offset);
if (ret <= 0) {
return ret;
}
io->offset += ret;
/* request payload is fully received */
if (io->offset == io->payload_size) {
io->offset = 0;
io->state = NBD_IO_PROCESS;
ret = spdk_nbd_io_exec(io);
}
}
return ret;
}
static int
spdk_nbd_io_xmit(struct nbd_io *io)
{
struct spdk_nbd_disk *nbd = io->nbd;
int ret = 0;
/* resp error and handler are already set in io_done */
if (io->state == NBD_IO_XMIT_RESP) {
ret = write_to_socket(nbd->spdk_sp_fd, (char *)&io->resp + io->offset,
sizeof(io->resp) - io->offset);
if (ret <= 0) {
return ret;
}
io->offset += ret;
/* response is fully transmitted */
if (io->offset == sizeof(io->resp)) {
io->offset = 0;
/* transmit payload only when NBD_CMD_READ with no resp error */
if (from_be32(&io->req.type) != NBD_CMD_READ || io->resp.error != 0) {
io->state = NBD_IO_RECV_REQ;
if (io->payload) {
spdk_dma_free(io->payload);
io->payload = NULL;
}
} else {
io->state = NBD_IO_XMIT_PAYLOAD;
}
}
}
if (io->state == NBD_IO_XMIT_PAYLOAD) {
ret = write_to_socket(nbd->spdk_sp_fd, io->payload + io->offset, io->payload_size - io->offset);
if (ret <= 0) {
return ret;
}
io->offset += ret;
/* read payload is fully transmitted */
if (io->offset == io->payload_size) {
io->offset = 0;
io->state = NBD_IO_RECV_REQ;
if (io->payload) {
spdk_dma_free(io->payload);
io->payload = NULL;
}
}
} }
return 0; return 0;
@ -424,69 +514,16 @@ static int
_spdk_nbd_poll(struct spdk_nbd_disk *nbd) _spdk_nbd_poll(struct spdk_nbd_disk *nbd)
{ {
struct nbd_io *io = &nbd->io; struct nbd_io *io = &nbd->io;
int fd = nbd->spdk_sp_fd; int rc;
int64_t ret;
int rc;
if (io->req_in_progress) { rc = spdk_nbd_io_recv(io);
ret = read_from_socket(fd, (char *)&io->req + io->offset, sizeof(io->req) - io->offset); if (rc < 0) {
if (ret <= 0) { return rc;
return ret;
}
io->offset += ret;
if (io->offset == sizeof(io->req)) {
io->req_in_progress = false;
rc = process_request(nbd);
if (rc != 0) {
return rc;
}
}
} }
if (io->payload_in_progress && is_write(io->type)) { rc = spdk_nbd_io_xmit(io);
ret = read_from_socket(fd, io->payload + io->offset, io->payload_size - io->offset);
if (ret <= 0) {
return ret;
}
io->offset += ret;
if (io->offset == io->payload_size) {
io->payload_in_progress = false;
nbd_submit_bdev_io(nbd->bdev, nbd->bdev_desc, nbd->ch, io);
io->offset = 0;
}
}
if (io->resp_in_progress) { return rc;
ret = write_to_socket(fd, (char *)&io->resp + io->offset, sizeof(io->resp) - io->offset);
if (ret <= 0) {
return ret;
}
io->offset += ret;
if (io->offset == sizeof(io->resp)) {
io->resp_in_progress = false;
if (is_read(io->type)) {
io->payload_in_progress = true;
} else {
io->req_in_progress = true;
}
io->offset = 0;
}
}
if (io->payload_in_progress && is_read(io->type)) {
ret = write_to_socket(fd, io->payload + io->offset, io->payload_size - io->offset);
if (ret <= 0) {
return ret;
}
io->offset += ret;
if (io->offset == io->payload_size) {
io->payload_in_progress = false;
io->req_in_progress = true;
io->offset = 0;
}
}
return 0;
} }
static void static void
@ -544,6 +581,8 @@ spdk_nbd_start(const char *bdev_name, const char *nbd_path)
if (nbd == NULL) { if (nbd == NULL) {
return NULL; return NULL;
} }
nbd->io.nbd = nbd;
nbd->dev_fd = -1; nbd->dev_fd = -1;
nbd->spdk_sp_fd = -1; nbd->spdk_sp_fd = -1;
nbd->kernel_sp_fd = -1; nbd->kernel_sp_fd = -1;
@ -640,7 +679,6 @@ spdk_nbd_start(const char *bdev_name, const char *nbd_path)
} }
to_be32(&nbd->io.resp.magic, NBD_REPLY_MAGIC); to_be32(&nbd->io.resp.magic, NBD_REPLY_MAGIC);
nbd->io.req_in_progress = true;
nbd->nbd_poller = spdk_poller_register(spdk_nbd_poll, nbd, 0); nbd->nbd_poller = spdk_poller_register(spdk_nbd_poll, nbd, 0);