From f69367c7881be9c5027e89ed4bc6e7a31c129753 Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Mon, 19 Apr 2021 11:36:29 -0700 Subject: [PATCH] fio_nvme: defer qpair allocation to file_open callback All jobs are created at boot, meaning the setup callback is invoked for all jobs before any are executed. But it may be useful to put 'stonewall' parameters in the job file to execute a bunch of workloads in succession, starting one workload when the previous one completes. But since qpairs are created currently during setup, the total number of workloads that can be expressed is limited since qpairs for all workloads are allocated up front. So instead defer allocation of the io qpairs until the file_open callback. These don't get called until the job associated with the 'file' (in this case, the nvme namespace) is ready to execute. Note that we cannot free the qpairs in the file_close callback, since fio may 'close' the file before all I/O have been completed. Signed-off-by: Jim Harris Change-Id: I3c60cf27c3660a3c94042c0de719f5bebdb9b417 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7481 Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto Reviewed-by: Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins --- examples/nvme/fio_plugin/fio_plugin.c | 45 +++++++++++++++------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/examples/nvme/fio_plugin/fio_plugin.c b/examples/nvme/fio_plugin/fio_plugin.c index e1eef0082..c2eee107f 100644 --- a/examples/nvme/fio_plugin/fio_plugin.c +++ b/examples/nvme/fio_plugin/fio_plugin.c @@ -319,7 +319,6 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, { struct thread_data *td = cb_ctx; struct spdk_fio_thread *fio_thread = td->io_ops_data; - struct spdk_nvme_io_qpair_opts qpopts; struct spdk_fio_ctrlr *fio_ctrlr; struct spdk_fio_qpair *fio_qpair; struct spdk_nvme_ns *ns; @@ -398,25 +397,7 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, return; } - spdk_nvme_ctrlr_get_default_io_qpair_opts(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts)); - qpopts.delay_cmd_submit = true; - if (fio_options->enable_wrr) { - qpopts.qprio = fio_options->wrr_priority; - } - - fio_qpair->qpair = spdk_nvme_ctrlr_alloc_io_qpair(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts)); - if (!fio_qpair->qpair) { - SPDK_ERRLOG("Cannot allocate nvme io_qpair any more\n"); - g_error = true; - free(fio_qpair); - return; - } - - if (fio_options->print_qid_mappings == 1) { - log_info("job %s: %s qid %d\n", td->o.name, f->file_name, - spdk_nvme_qpair_get_id(fio_qpair->qpair)); - } - + f->engine_data = fio_qpair; fio_qpair->ns = ns; fio_qpair->f = f; fio_qpair->fio_ctrlr = fio_ctrlr; @@ -677,6 +658,30 @@ static int spdk_fio_setup(struct thread_data *td) static int spdk_fio_open(struct thread_data *td, struct fio_file *f) { + struct spdk_fio_qpair *fio_qpair = f->engine_data; + struct spdk_fio_ctrlr *fio_ctrlr = fio_qpair->fio_ctrlr; + struct spdk_fio_options *fio_options = td->eo; + struct spdk_nvme_io_qpair_opts qpopts; + + spdk_nvme_ctrlr_get_default_io_qpair_opts(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts)); + qpopts.delay_cmd_submit = true; + if (fio_options->enable_wrr) { + qpopts.qprio = fio_options->wrr_priority; + } + + fio_qpair->qpair = spdk_nvme_ctrlr_alloc_io_qpair(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts)); + if (!fio_qpair->qpair) { + SPDK_ERRLOG("Cannot allocate nvme io_qpair any more\n"); + g_error = true; + free(fio_qpair); + return -1; + } + + if (fio_options->print_qid_mappings == 1) { + log_info("job %s: %s qid %d\n", td->o.name, f->file_name, + spdk_nvme_qpair_get_id(fio_qpair->qpair)); + } + return 0; }