ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fio_plugin: Support multiple jobs for fio when enabling SPDK

Browse Source 
Change-Id: I455c6075d93def33647881cc62a2856b1d014b6d
Signed-off-by: Ziye Yang <ziye.yang@intel.com>
This commit is contained in:
Ziye Yang 2017-04-13 13:27:25 +08:00 committed by Daniel Verkamp
parent decea46f59
commit 1d9891989c
3 changed files with 110 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/nvme/fio_plugin/README.md
View File

@ -42,8 +42,8 @@ NVMe namespaces start at 1, not 0! And it should be put on the end. For example,
NVMe device should use this format: domain.bus.slot.func NVMe device should use this format: domain.bus.slot.func
2. For devices exported by NVMe-oF target, 'trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1' 2. For devices exported by NVMe-oF target, 'trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1'
Currently the SPDK fio plugin is limited to a single thread, so only one job is supported. Currently the SPDK fio plugin is limited to thread usage model, so fio jobs must also specify thread=1
fio jobs must also specify thread=1 when using the SPDK fio plugin. when using the SPDK fio plugin.
When testing random workloads, it is recommended to set norandommap=1. fio's random map When testing random workloads, it is recommended to set norandommap=1. fio's random map
processing consumes extra CPU cycles which will degrade performance over time with processing consumes extra CPU cycles which will degrade performance over time with

8
examples/nvme/fio_plugin/example_config.fio
View File

@ -6,10 +6,10 @@ direct=1
verify=0 verify=0
time_based=1 time_based=1
ramp_time=0 ramp_time=0
runtime=1 runtime=2
[test]
iodepth=128 iodepth=128
rw=randrw rw=randrw
bs=4k bs=4k
numjobs=1
[test]
numjobs=2

198
examples/nvme/fio_plugin/fio_plugin.c
View File

@ -58,27 +58,28 @@ struct spdk_fio_request {
struct spdk_fio_thread *fio_thread; struct spdk_fio_thread *fio_thread;
}; };
struct spdk_fio_ns {
struct fio_file *f;
struct spdk_nvme_ns *ns;
struct spdk_fio_ns *next;
};
struct spdk_fio_ctrlr { struct spdk_fio_ctrlr {
struct spdk_nvme_transport_id tr_id; struct spdk_nvme_transport_id tr_id;
struct spdk_nvme_ctrlr_opts opts;
struct spdk_nvme_ctrlr *ctrlr; struct spdk_nvme_ctrlr *ctrlr;
struct spdk_fio_ctrlr *next; struct spdk_fio_ctrlr *next;
};
struct spdk_nvme_qpair *qpair; struct spdk_fio_ctrlr *ctrlr_g;
int td_count;
pthread_mutex_t mutex;
struct spdk_fio_ns *ns_list; struct spdk_fio_qpair {
struct fio_file *f;
struct spdk_nvme_qpair *qpair;
struct spdk_nvme_ns *ns;
struct spdk_fio_qpair *next;
}; };
struct spdk_fio_thread { struct spdk_fio_thread {
struct thread_data *td; struct thread_data *td;
struct spdk_fio_ctrlr *ctrlr_list; struct spdk_fio_qpair *fio_qpair;
struct io_u **iocq; // io completion queue struct io_u **iocq; // io completion queue
unsigned int iocq_count; // number of iocq entries filled by last getevents unsigned int iocq_count; // number of iocq entries filled by last getevents
@ -94,6 +95,21 @@ probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
return true; return true;
} }
static struct spdk_fio_ctrlr *
get_fio_ctrlr(const struct spdk_nvme_transport_id *trid)
{
struct spdk_fio_ctrlr *fio_ctrlr = ctrlr_g;
while (fio_ctrlr) {
if (spdk_nvme_transport_id_compare(trid, &fio_ctrlr->tr_id) == 0) {
return fio_ctrlr;
}
fio_ctrlr = fio_ctrlr->next;
}
return NULL;
}
static void static void
attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
@ -101,10 +117,10 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
struct thread_data *td = cb_ctx; struct thread_data *td = cb_ctx;
struct spdk_fio_thread *fio_thread = td->io_ops_data; struct spdk_fio_thread *fio_thread = td->io_ops_data;
struct spdk_fio_ctrlr *fio_ctrlr; struct spdk_fio_ctrlr *fio_ctrlr;
struct spdk_fio_ns *fio_ns; struct spdk_fio_qpair *fio_qpair;
struct spdk_nvme_ns *ns;
struct fio_file *f = fio_thread->current_f; struct fio_file *f = fio_thread->current_f;
uint32_t ns_id; uint32_t ns_id;
bool ctrlr_is_added = false;
char *p; char *p;
p = strstr(f->file_name, "ns="); p = strstr(f->file_name, "ns=");
@ -115,68 +131,56 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
return; return;
} }
/* check whether this trid is already added */ fio_ctrlr = get_fio_ctrlr(trid);
fio_ctrlr = fio_thread->ctrlr_list;
while (fio_ctrlr) {
if (spdk_nvme_transport_id_compare(trid, &fio_ctrlr->tr_id) == 0) {
ctrlr_is_added = true;
break;
}
fio_ctrlr = fio_ctrlr->next;
}
/* it is a new ctrlr and needs to be added */ /* it is a new ctrlr and needs to be added */
if (!ctrlr_is_added) { if (!fio_ctrlr) {
/* Create an fio_ctrlr and add it to the list */ /* Create an fio_ctrlr and add it to the list */
fio_ctrlr = calloc(1, sizeof(*fio_ctrlr)); fio_ctrlr = calloc(1, sizeof(*fio_ctrlr));
if (!fio_ctrlr) { if (!fio_ctrlr) {
SPDK_ERRLOG("Cannot allocate space for fio_ctrlr\n"); SPDK_ERRLOG("Cannot allocate space for fio_ctrlr\n");
return; return;
} }
fio_ctrlr->opts = *opts;
fio_ctrlr->ctrlr = ctrlr; fio_ctrlr->ctrlr = ctrlr;
fio_ctrlr->tr_id = *trid; fio_ctrlr->tr_id = *trid;
fio_ctrlr->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, 0); fio_ctrlr->next = ctrlr_g;
fio_ctrlr->ns_list = NULL; ctrlr_g = fio_ctrlr;
fio_ctrlr->next = fio_thread->ctrlr_list;
fio_thread->ctrlr_list = fio_ctrlr;
} }
/* check whether this name space is existing or not */ ns = spdk_nvme_ctrlr_get_ns(fio_ctrlr->ctrlr, ns_id);
fio_ns = fio_ctrlr->ns_list; if (ns == NULL) {
while (fio_ns) { SPDK_ERRLOG("Cannot get namespace by ns_id=%d\n", ns_id);
if (spdk_nvme_ns_get_id(fio_ns->ns) == ns_id) { return;
}
fio_qpair = fio_thread->fio_qpair;
while (fio_qpair != NULL) {
if (fio_qpair->f == f) {
return; return;
} }
fio_ns = fio_ns->next; fio_qpair = fio_qpair->next;
} }
/* create a new namespace */ /* create a new qpair */
fio_ns = calloc(1, sizeof(*fio_ns)); fio_qpair = calloc(1, sizeof(*fio_qpair));
if (fio_ns == NULL) { if (!fio_qpair) {
SPDK_ERRLOG("Cannot allocate space for fio_ns\n"); SPDK_ERRLOG("Cannot allocate space for fio_qpair\n");
return;
}
fio_ns->f = f;
fio_ns->ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
if (fio_ns->ns == NULL) {
SPDK_ERRLOG("Cannot get namespace by ns_id=%d\n", ns_id);
free(fio_ns);
return; return;
} }
fio_qpair->qpair = spdk_nvme_ctrlr_alloc_io_qpair(fio_ctrlr->ctrlr, 0);
fio_qpair->ns = ns;
fio_qpair->f = f;
fio_qpair->next = fio_thread->fio_qpair;
fio_thread->fio_qpair = fio_qpair;
f->real_file_size = spdk_nvme_ns_get_size(fio_ns->ns); f->real_file_size = spdk_nvme_ns_get_size(fio_qpair->ns);
if (f->real_file_size <= 0) { if (f->real_file_size <= 0) {
SPDK_ERRLOG("Cannot get namespace size by ns=%p\n", fio_ns->ns); SPDK_ERRLOG("Cannot get namespace size by ns=%p\n", ns);
free(fio_ns);
return; return;
} }
f->filetype = FIO_TYPE_BLOCK; f->filetype = FIO_TYPE_BLOCK;
fio_file_set_size_known(f); fio_file_set_size_known(f);
fio_ns->next = fio_ctrlr->ns_list;
fio_ctrlr->ns_list = fio_ns;
} }
static void static void
@ -209,6 +213,7 @@ static int spdk_fio_setup(struct thread_data *td)
char *p; char *p;
int rc; int rc;
struct spdk_nvme_transport_id trid; struct spdk_nvme_transport_id trid;
struct spdk_fio_ctrlr *fio_ctrlr;
char *trid_info; char *trid_info;
unsigned int i; unsigned int i;
@ -234,6 +239,7 @@ static int spdk_fio_setup(struct thread_data *td)
spdk_env_init(&opts); spdk_env_init(&opts);
spdk_env_initialized = true; spdk_env_initialized = true;
cpu_core_unaffinitized(); cpu_core_unaffinitized();
pthread_mutex_init(&mutex, NULL);
} }
for_each_file(td, f, i) { for_each_file(td, f, i) {
@ -278,13 +284,20 @@ static int spdk_fio_setup(struct thread_data *td)
fio_thread->current_f = f; fio_thread->current_f = f;
/* Enumerate all of the controllers */ fio_ctrlr = get_fio_ctrlr(&trid);
if (spdk_nvme_probe(&trid, td, probe_cb, attach_cb, NULL) != 0) { if (fio_ctrlr) {
SPDK_ERRLOG("spdk_nvme_probe() failed\n"); attach_cb(td, &trid, fio_ctrlr->ctrlr, &fio_ctrlr->opts);
continue; } else {
/* Enumerate all of the controllers */
if (spdk_nvme_probe(&trid, td, probe_cb, attach_cb, NULL) != 0) {
SPDK_ERRLOG("spdk_nvme_probe() failed\n");
continue;
}
} }
} }
td_count++;
return 0; return 0;
} }
@ -351,42 +364,33 @@ static int spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
int rc = 1; int rc = 1;
struct spdk_fio_thread *fio_thread = td->io_ops_data; struct spdk_fio_thread *fio_thread = td->io_ops_data;
struct spdk_fio_request *fio_req = io_u->engine_data; struct spdk_fio_request *fio_req = io_u->engine_data;
struct spdk_fio_ctrlr *fio_ctrlr; struct spdk_fio_qpair *fio_qpair;
struct spdk_fio_ns *fio_ns; struct spdk_nvme_ns *ns = NULL;
bool found_ns = false;
/* Find the namespace that corresponds to the file in the io_u */ /* Find the namespace that corresponds to the file in the io_u */
fio_ctrlr = fio_thread->ctrlr_list; fio_qpair = fio_thread->fio_qpair;
while (fio_ctrlr != NULL) { while (fio_qpair != NULL) {
fio_ns = fio_ctrlr->ns_list; if (fio_qpair->f == io_u->file) {
while (fio_ns != NULL) { ns = fio_qpair->ns;
if (fio_ns->f == io_u->file) {
found_ns = true;
break;
}
fio_ns = fio_ns->next;
}
if (found_ns) {
break; break;
} }
fio_ctrlr = fio_ctrlr->next; fio_qpair = fio_qpair->next;
} }
if (fio_ctrlr == NULL || fio_ns == NULL) { if (fio_qpair == NULL || ns == NULL) {
return FIO_Q_COMPLETED; return FIO_Q_COMPLETED;
} }
assert(found_ns == true);
uint32_t block_size = spdk_nvme_ns_get_sector_size(fio_ns->ns); uint32_t block_size = spdk_nvme_ns_get_sector_size(ns);
uint64_t lba = io_u->offset / block_size; uint64_t lba = io_u->offset / block_size;
uint32_t lba_count = io_u->xfer_buflen / block_size; uint32_t lba_count = io_u->xfer_buflen / block_size;
switch (io_u->ddir) { switch (io_u->ddir) {
case DDIR_READ: case DDIR_READ:
rc = spdk_nvme_ns_cmd_read(fio_ns->ns, fio_ctrlr->qpair, io_u->buf, lba, lba_count, rc = spdk_nvme_ns_cmd_read(ns, fio_qpair->qpair, io_u->buf, lba, lba_count,
spdk_fio_completion_cb, fio_req, 0); spdk_fio_completion_cb, fio_req, 0);
break; break;
case DDIR_WRITE: case DDIR_WRITE:
rc = spdk_nvme_ns_cmd_write(fio_ns->ns, fio_ctrlr->qpair, io_u->buf, lba, lba_count, rc = spdk_nvme_ns_cmd_write(ns, fio_qpair->qpair, io_u->buf, lba, lba_count,
spdk_fio_completion_cb, fio_req, 0); spdk_fio_completion_cb, fio_req, 0);
break; break;
default: default:
@ -412,7 +416,7 @@ static int spdk_fio_getevents(struct thread_data *td, unsigned int min,
unsigned int max, const struct timespec *t) unsigned int max, const struct timespec *t)
{ {
struct spdk_fio_thread *fio_thread = td->io_ops_data; struct spdk_fio_thread *fio_thread = td->io_ops_data;
struct spdk_fio_ctrlr *fio_ctrlr; struct spdk_fio_qpair *fio_qpair;
struct timespec t0, t1; struct timespec t0, t1;
uint64_t timeout = 0; uint64_t timeout = 0;
@ -424,15 +428,15 @@ static int spdk_fio_getevents(struct thread_data *td, unsigned int min,
fio_thread->iocq_count = 0; fio_thread->iocq_count = 0;
for (;;) { for (;;) {
fio_ctrlr = fio_thread->ctrlr_list; fio_qpair = fio_thread->fio_qpair;
while (fio_ctrlr != NULL) { while (fio_qpair != NULL) {
spdk_nvme_qpair_process_completions(fio_ctrlr->qpair, max - fio_thread->iocq_count); spdk_nvme_qpair_process_completions(fio_qpair->qpair, max - fio_thread->iocq_count);
if (fio_thread->iocq_count >= min) { if (fio_thread->iocq_count >= min) {
return fio_thread->iocq_count; return fio_thread->iocq_count;
} }
fio_ctrlr = fio_ctrlr->next; fio_qpair = fio_qpair->next;
} }
if (t) { if (t) {
@ -457,25 +461,31 @@ static int spdk_fio_invalidate(struct thread_data *td, struct fio_file *f)
static void spdk_fio_cleanup(struct thread_data *td) static void spdk_fio_cleanup(struct thread_data *td)
{ {
struct spdk_fio_thread *fio_thread = td->io_ops_data; struct spdk_fio_thread *fio_thread = td->io_ops_data;
struct spdk_fio_ctrlr *fio_ctrlr, *fio_ctrlr_tmp; struct spdk_fio_qpair *fio_qpair, *fio_qpair_tmp;
struct spdk_fio_ns *fio_ns, *fio_ns_tmp;
fio_ctrlr = fio_thread->ctrlr_list; fio_qpair = fio_thread->fio_qpair;
while (fio_ctrlr != NULL) { while (fio_qpair != NULL) {
fio_ns = fio_ctrlr->ns_list; spdk_nvme_ctrlr_free_io_qpair(fio_qpair->qpair);
while (fio_ns != NULL) { fio_qpair_tmp = fio_qpair->next;
fio_ns_tmp = fio_ns->next; free(fio_qpair);
free(fio_ns); fio_qpair = fio_qpair_tmp;
fio_ns = fio_ns_tmp;
}
spdk_nvme_ctrlr_free_io_qpair(fio_ctrlr->qpair);
spdk_nvme_detach(fio_ctrlr->ctrlr);
fio_ctrlr_tmp = fio_ctrlr->next;
free(fio_ctrlr);
fio_ctrlr = fio_ctrlr_tmp;
} }
free(fio_thread); free(fio_thread);
pthread_mutex_lock(&mutex);
td_count--;
if (td_count == 0) {
struct spdk_fio_ctrlr *fio_ctrlr, *fio_ctrlr_tmp;
fio_ctrlr = ctrlr_g;
while (fio_ctrlr != NULL) {
spdk_nvme_detach(fio_ctrlr->ctrlr);
fio_ctrlr_tmp = fio_ctrlr->next;
free(fio_ctrlr);
fio_ctrlr = fio_ctrlr_tmp;
}
}
pthread_mutex_unlock(&mutex);
} }
/* FIO imports this structure using dlsym */ /* FIO imports this structure using dlsym */