From 503cddac709ac385f506ccddf33b2284fe060fd6 Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Mon, 11 Jun 2018 05:35:11 -0700 Subject: [PATCH] bdev: add global structure for bdev_io options Currently this just includes the overall size of the spdk_bdev_io pool and the size of the per-thread spdk_bdev_io caches. Later patches will allow configuring these via INI config file and JSON-RPC. Signed-off-by: Jim Harris Change-Id: I7e235ee6d2d7123d8460eeacde999c7b51017c43 Reviewed-on: https://review.gerrithub.io/414710 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Ben Walker Reviewed-by: Dariusz Stojaczyk Reviewed-by: Shuhei Matsumoto --- include/spdk/bdev.h | 9 +++++++++ lib/bdev/bdev.c | 36 ++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/include/spdk/bdev.h b/include/spdk/bdev.h index 3ccc1d400..39fb72233 100644 --- a/include/spdk/bdev.h +++ b/include/spdk/bdev.h @@ -128,6 +128,15 @@ struct spdk_bdev_io_stat { uint64_t ticks_rate; }; +struct spdk_bdev_opts { + uint32_t bdev_io_pool_size; + uint32_t bdev_io_cache_size; +}; + +void spdk_bdev_get_opts(struct spdk_bdev_opts *opts); + +int spdk_bdev_set_opts(struct spdk_bdev_opts *opts); + /** * Block device initialization callback. * diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index a35330aeb..df867ceda 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -104,6 +104,11 @@ static struct spdk_bdev_mgr g_bdev_mgr = { .module_init_complete = false, }; +static struct spdk_bdev_opts g_bdev_opts = { + .bdev_io_pool_size = SPDK_BDEV_IO_POOL_SIZE, + .bdev_io_cache_size = SPDK_BDEV_IO_CACHE_SIZE, +}; + static spdk_bdev_init_cb g_init_cb_fn = NULL; static void *g_init_cb_arg = NULL; @@ -158,6 +163,7 @@ struct spdk_bdev_mgmt_channel { */ bdev_io_stailq_t per_thread_cache; uint32_t per_thread_cache_count; + uint32_t bdev_io_cache_size; TAILQ_HEAD(, spdk_bdev_shared_resource) shared_resources; }; @@ -249,6 +255,26 @@ struct spdk_bdev_iostat_ctx { static void spdk_bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +void +spdk_bdev_get_opts(struct spdk_bdev_opts *opts) +{ + *opts = g_bdev_opts; +} + +int +spdk_bdev_set_opts(struct spdk_bdev_opts *opts) +{ + if (opts->bdev_io_pool_size < opts->bdev_io_cache_size * spdk_thread_get_count()) { + SPDK_ERRLOG("bdev_io_pool_size %" PRIu32 " is not compatible with bdev_io_cache_size %" PRIu32 + " and %" PRIu32 " threads\n", opts->bdev_io_pool_size, opts->bdev_io_cache_size, + spdk_thread_get_count()); + return -1; + } + + g_bdev_opts = *opts; + return 0; +} + struct spdk_bdev * spdk_bdev_first(void) { @@ -483,6 +509,7 @@ spdk_bdev_mgmt_channel_create(void *io_device, void *ctx_buf) STAILQ_INIT(&ch->per_thread_cache); ch->per_thread_cache_count = 0; + ch->bdev_io_cache_size = g_bdev_opts.bdev_io_cache_size; TAILQ_INIT(&ch->shared_resources); @@ -606,6 +633,7 @@ spdk_bdev_modules_init(void) g_bdev_mgr.module_init_complete = true; return rc; } + void spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg) { @@ -621,7 +649,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg) snprintf(mempool_name, sizeof(mempool_name), "bdev_io_%d", getpid()); g_bdev_mgr.bdev_io_pool = spdk_mempool_create(mempool_name, - SPDK_BDEV_IO_POOL_SIZE, + g_bdev_opts.bdev_io_pool_size, sizeof(struct spdk_bdev_io) + spdk_bdev_module_get_max_ctx_size(), 0, @@ -697,10 +725,10 @@ spdk_bdev_mgr_unregister_cb(void *io_device) { spdk_bdev_fini_cb cb_fn = g_fini_cb_fn; - if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != SPDK_BDEV_IO_POOL_SIZE) { + if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != g_bdev_opts.bdev_io_pool_size) { SPDK_ERRLOG("bdev IO pool count is %zu but should be %u\n", spdk_mempool_count(g_bdev_mgr.bdev_io_pool), - SPDK_BDEV_IO_POOL_SIZE); + g_bdev_opts.bdev_io_pool_size); } if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != BUF_SMALL_POOL_SIZE) { @@ -862,7 +890,7 @@ spdk_bdev_put_io(struct spdk_bdev_io *bdev_io) spdk_bdev_io_put_buf(bdev_io); } - if (ch->per_thread_cache_count < SPDK_BDEV_IO_CACHE_SIZE) { + if (ch->per_thread_cache_count < ch->bdev_io_cache_size) { ch->per_thread_cache_count++; STAILQ_INSERT_TAIL(&ch->per_thread_cache, bdev_io, internal.buf_link); } else {