ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bdev: Make the small and large buffer pool configurable.

Browse Source 
This patch includes the following work:

1 Add two fields in spdk_bdev_opts structure and make the
two fields configurable.
2 Update the unit test in bdev_ut.c
3 Revise the set_bdev_options rpc call and add the support
to use the two new fields.

Change-Id: Idd6073b3a3ffaba8161e3ffa9444a9e533e67f6d
Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5664
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Ziye Yang 2020-12-22 23:40:59 +08:00 committed by Tomasz Zawadzki
parent 590b6e7507
commit f420b9ef0a
8 changed files with 142 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -6,7 +6,9 @@
An `opts_size`element was added in the `spdk_bdev_opts` structure to solve the
ABI compatiblity issue between different SPDK version. And also add `opts_size`
parameter in spdk_bdev_get_opts function.
parameter in spdk_bdev_get_opts function. Two fields `small_buf_pool_size` and
`large_buf_pool_size` were added into spdk_bdev_opts, which were used to determine
the small and large buffer pool size of the whole bdev module.
### event
@ -66,6 +68,9 @@ listening on portals for a portal group until all associated target nodes are cr
at startup, otherwise some iSCSI initiators may fail to re-login when SPDK iSCSI
target application restarts.
Two optional parameter `--small-buf-pool-size` and `--large-buf-pool-size` were added
into `bdev_set_options` function.
## ioat
The PCI BDF whitelist option has been removed from the ioat_scan_accel_engine RPC.

3
include/spdk/bdev.h
View File

@ -194,6 +194,9 @@ struct spdk_bdev_opts {
* New added fields should be put at the end of the struct.
*/
size_t opts_size;
uint32_t small_buf_pool_size;
uint32_t large_buf_pool_size;
};
/**

30
lib/bdev/bdev.c
View File

@ -134,6 +134,8 @@ static struct spdk_bdev_opts g_bdev_opts = {
.bdev_io_pool_size = SPDK_BDEV_IO_POOL_SIZE,
.bdev_io_cache_size = SPDK_BDEV_IO_CACHE_SIZE,
.bdev_auto_examine = SPDK_BDEV_AUTO_EXAMINE,
.small_buf_pool_size = BUF_SMALL_POOL_SIZE,
.large_buf_pool_size = BUF_LARGE_POOL_SIZE,
};
static spdk_bdev_init_cb g_init_cb_fn = NULL;
@ -393,10 +395,12 @@ spdk_bdev_get_opts(struct spdk_bdev_opts *opts, size_t opts_size)
SET_FIELD(bdev_io_pool_size);
SET_FIELD(bdev_io_cache_size);
SET_FIELD(bdev_auto_examine);
SET_FIELD(small_buf_pool_size);
SET_FIELD(large_buf_pool_size);
/* Do not remove this statement, you should always update this statement when you adding a new field,
* and do not forget to add the SET_FIELD statement for your added field. */
SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_opts) == 24, "Incorrect size");
SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_opts) == 32, "Incorrect size");
#undef SET_FIELD
}
@ -430,6 +434,16 @@ spdk_bdev_set_opts(struct spdk_bdev_opts *opts)
return -1;
}
if (opts->small_buf_pool_size < BUF_SMALL_POOL_SIZE) {
SPDK_ERRLOG("small_buf_pool_size must be at least %" PRIu32 "\n", BUF_SMALL_POOL_SIZE);
return -1;
}
if (opts->large_buf_pool_size < BUF_LARGE_POOL_SIZE) {
SPDK_ERRLOG("large_buf_pool_size must be at least %" PRIu32 "\n", BUF_LARGE_POOL_SIZE);
return -1;
}
#define SET_FIELD(field) \
if (offsetof(struct spdk_bdev_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
g_bdev_opts.field = opts->field; \
@ -438,6 +452,8 @@ spdk_bdev_set_opts(struct spdk_bdev_opts *opts)
SET_FIELD(bdev_io_pool_size);
SET_FIELD(bdev_io_cache_size);
SET_FIELD(bdev_auto_examine);
SET_FIELD(small_buf_pool_size);
SET_FIELD(large_buf_pool_size);
g_bdev_opts.opts_size = opts->opts_size;
@ -1305,7 +1321,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg)
snprintf(mempool_name, sizeof(mempool_name), "buf_small_pool_%d", getpid());
g_bdev_mgr.buf_small_pool = spdk_mempool_create(mempool_name,
BUF_SMALL_POOL_SIZE,
g_bdev_opts.small_buf_pool_size,
SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +
SPDK_BDEV_POOL_ALIGNMENT,
cache_size,
@ -1320,7 +1336,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg)
snprintf(mempool_name, sizeof(mempool_name), "buf_large_pool_%d", getpid());
g_bdev_mgr.buf_large_pool = spdk_mempool_create(mempool_name,
BUF_LARGE_POOL_SIZE,
g_bdev_opts.large_buf_pool_size,
SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_LARGE_BUF_MAX_SIZE) +
SPDK_BDEV_POOL_ALIGNMENT,
cache_size,
@ -1374,10 +1390,10 @@ bdev_mgr_unregister_cb(void *io_device)
}
if (g_bdev_mgr.buf_small_pool) {
if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != BUF_SMALL_POOL_SIZE) {
if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != g_bdev_opts.small_buf_pool_size) {
SPDK_ERRLOG("Small buffer pool count is %zu but should be %u\n",
spdk_mempool_count(g_bdev_mgr.buf_small_pool),
BUF_SMALL_POOL_SIZE);
g_bdev_opts.small_buf_pool_size);
assert(false);
}
@ -1385,10 +1401,10 @@ bdev_mgr_unregister_cb(void *io_device)
}
if (g_bdev_mgr.buf_large_pool) {
if (spdk_mempool_count(g_bdev_mgr.buf_large_pool) != BUF_LARGE_POOL_SIZE) {
if (spdk_mempool_count(g_bdev_mgr.buf_large_pool) != g_bdev_opts.large_buf_pool_size) {
SPDK_ERRLOG("Large buffer pool count is %zu but should be %u\n",
spdk_mempool_count(g_bdev_mgr.buf_large_pool),
BUF_LARGE_POOL_SIZE);
g_bdev_opts.large_buf_pool_size);
assert(false);
}

13
lib/bdev/bdev_rpc.c
View File

@ -46,12 +46,16 @@ struct spdk_rpc_set_bdev_opts {
uint32_t bdev_io_pool_size;
uint32_t bdev_io_cache_size;
bool bdev_auto_examine;
uint32_t small_buf_pool_size;
uint32_t large_buf_pool_size;
};
static const struct spdk_json_object_decoder rpc_set_bdev_opts_decoders[] = {
{"bdev_io_pool_size", offsetof(struct spdk_rpc_set_bdev_opts, bdev_io_pool_size), spdk_json_decode_uint32, true},
{"bdev_io_cache_size", offsetof(struct spdk_rpc_set_bdev_opts, bdev_io_cache_size), spdk_json_decode_uint32, true},
{"bdev_auto_examine", offsetof(struct spdk_rpc_set_bdev_opts, bdev_auto_examine), spdk_json_decode_bool, true},
{"small_buf_pool_size", offsetof(struct spdk_rpc_set_bdev_opts, small_buf_pool_size), spdk_json_decode_uint32, true},
{"large_buf_pool_size", offsetof(struct spdk_rpc_set_bdev_opts, large_buf_pool_size), spdk_json_decode_uint32, true},
};
static void
@ -63,6 +67,8 @@ rpc_bdev_set_options(struct spdk_jsonrpc_request *request, const struct spdk_jso
rpc_opts.bdev_io_pool_size = UINT32_MAX;
rpc_opts.bdev_io_cache_size = UINT32_MAX;
rpc_opts.small_buf_pool_size = UINT32_MAX;
rpc_opts.large_buf_pool_size = UINT32_MAX;
rpc_opts.bdev_auto_examine = true;
if (params != NULL) {
@ -83,6 +89,13 @@ rpc_bdev_set_options(struct spdk_jsonrpc_request *request, const struct spdk_jso
bdev_opts.bdev_io_cache_size = rpc_opts.bdev_io_cache_size;
}
bdev_opts.bdev_auto_examine = rpc_opts.bdev_auto_examine;
if (rpc_opts.small_buf_pool_size != UINT32_MAX) {
bdev_opts.small_buf_pool_size = rpc_opts.small_buf_pool_size;
}
if (rpc_opts.large_buf_pool_size != UINT32_MAX) {
bdev_opts.large_buf_pool_size = rpc_opts.large_buf_pool_size;
}
rc = spdk_bdev_set_opts(&bdev_opts);
if (rc != 0) {

6
scripts/rpc.py
View File

@ -171,12 +171,16 @@ if __name__ == "__main__":
rpc.bdev.bdev_set_options(args.client,
bdev_io_pool_size=args.bdev_io_pool_size,
bdev_io_cache_size=args.bdev_io_cache_size,
bdev_auto_examine=args.bdev_auto_examine)
bdev_auto_examine=args.bdev_auto_examine,
small_buf_pool_size=args.small_buf_pool_size,
large_buf_pool_size=args.large_buf_pool_size)
p = subparsers.add_parser('bdev_set_options', aliases=['set_bdev_options'],
help="""Set options of bdev subsystem""")
p.add_argument('-p', '--bdev-io-pool-size', help='Number of bdev_io structures in shared buffer pool', type=int)
p.add_argument('-c', '--bdev-io-cache-size', help='Maximum number of bdev_io structures cached per thread', type=int)
p.add_argument('-s', '--small-buf-pool-size', help='Maximum number of small buf (i.e., 8KB) pool size', type=int)
p.add_argument('-l', '--large-buf-pool-size', help='Maximum number of large buf (i.e., 64KB) pool size', type=int)
group = p.add_mutually_exclusive_group()
group.add_argument('-e', '--enable-auto-examine', dest='bdev_auto_examine', help='Allow to auto examine', action='store_true')
group.add_argument('-d', '--disable-auto-examine', dest='bdev_auto_examine', help='Not allow to auto examine', action='store_false')

10
scripts/rpc/bdev.py
View File

@ -2,13 +2,16 @@ from .helpers import deprecated_alias
@deprecated_alias('set_bdev_options')
def bdev_set_options(client, bdev_io_pool_size=None, bdev_io_cache_size=None, bdev_auto_examine=None):
def bdev_set_options(client, bdev_io_pool_size=None, bdev_io_cache_size=None, bdev_auto_examine=None,
small_buf_pool_size=None, large_buf_pool_size=None):
"""Set parameters for the bdev subsystem.
Args:
bdev_io_pool_size: number of bdev_io structures in shared buffer pool (optional)
bdev_io_cache_size: maximum number of bdev_io structures cached per thread (optional)
bdev_auto_examine: if set to false, the bdev layer will not examine every disks automatically (optional)
small_buf_pool_size: maximum number of small buffer (8KB buffer) pool size (optional)
large_buf_pool_size: maximum number of large buffer (64KB buffer) pool size (optional)
"""
params = {}
@ -18,7 +21,10 @@ def bdev_set_options(client, bdev_io_pool_size=None, bdev_io_cache_size=None, bd
params['bdev_io_cache_size'] = bdev_io_cache_size
if bdev_auto_examine is not None:
params["bdev_auto_examine"] = bdev_auto_examine
if small_buf_pool_size:
params['small_buf_pool_size'] = small_buf_pool_size
if large_buf_pool_size:
params['large_buf_pool_size'] = large_buf_pool_size
return client.call('bdev_set_options', params)

2
test/blobstore/blob_io_wait/blob_io_wait.sh
View File

@ -20,7 +20,7 @@ trap 'rm -f $testdir/bdevperf.json; rm -f $testdir/aio.bdev; killprocess $bdev_s
waitforlisten $bdev_svc_pid
# Minimal number of bdev io pool (128) and cache (1)
$rpc_py bdev_set_options --bdev-io-pool-size 128 --bdev-io-cache-size 1
$rpc_py bdev_set_options --bdev-io-pool-size 128 --bdev-io-cache-size 1 --small-buf-pool-size 8192 --large-buf-pool-size 1024
$rpc_py framework_start_init
$rpc_py bdev_aio_create $testdir/aio.bdev aio0 4096
$rpc_py bdev_lvol_create_lvstore aio0 lvs0

126
test/unit/lib/bdev/bdev.c/bdev_ut.c
View File

@ -898,13 +898,13 @@ bdev_io_types_test(void)
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc = NULL;
struct spdk_io_channel *io_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 4,
.bdev_io_cache_size = 2,
};
struct spdk_bdev_opts bdev_opts = {};
int rc;
bdev_opts.opts_size = sizeof(bdev_opts);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 4;
bdev_opts.bdev_io_cache_size = 2;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
spdk_bdev_initialize(bdev_init_cb, NULL);
@ -941,15 +941,15 @@ bdev_io_wait_test(void)
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc = NULL;
struct spdk_io_channel *io_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 4,
.bdev_io_cache_size = 2,
};
struct spdk_bdev_opts bdev_opts = {};
struct bdev_ut_io_wait_entry io_wait_entry;
struct bdev_ut_io_wait_entry io_wait_entry2;
int rc;
bdev_opts.opts_size = sizeof(bdev_opts);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 4;
bdev_opts.bdev_io_cache_size = 2;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
spdk_bdev_initialize(bdev_init_cb, NULL);
@ -1083,17 +1083,17 @@ bdev_io_boundary_split_test(void)
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc = NULL;
struct spdk_io_channel *io_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 512,
.bdev_io_cache_size = 64,
};
struct spdk_bdev_opts bdev_opts = {};
struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2];
struct ut_expected_io *expected_io;
void *md_buf = (void *)0xFF000000;
uint64_t i;
int rc;
bdev_opts.opts_size = sizeof(bdev_opts);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 512;
bdev_opts.bdev_io_cache_size = 64;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
spdk_bdev_initialize(bdev_init_cb, NULL);
@ -1724,15 +1724,16 @@ bdev_io_max_size_and_segment_split_test(void)
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc = NULL;
struct spdk_io_channel *io_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 512,
.bdev_io_cache_size = 64,
};
struct spdk_bdev_opts bdev_opts = {};
struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2];
struct ut_expected_io *expected_io;
uint64_t i;
int rc;
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 512;
bdev_opts.bdev_io_cache_size = 64;
bdev_opts.opts_size = sizeof(bdev_opts);
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
@ -2271,16 +2272,16 @@ bdev_io_mix_split_test(void)
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc = NULL;
struct spdk_io_channel *io_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 512,
.bdev_io_cache_size = 64,
};
struct spdk_bdev_opts bdev_opts = {};
struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2];
struct ut_expected_io *expected_io;
uint64_t i;
int rc;
bdev_opts.opts_size = sizeof(bdev_opts);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 512;
bdev_opts.bdev_io_cache_size = 64;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
spdk_bdev_initialize(bdev_init_cb, NULL);
@ -2542,15 +2543,15 @@ bdev_io_split_with_io_wait(void)
struct spdk_io_channel *io_ch;
struct spdk_bdev_channel *channel;
struct spdk_bdev_mgmt_channel *mgmt_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 2,
.bdev_io_cache_size = 1,
};
struct spdk_bdev_opts bdev_opts = {};
struct iovec iov[3];
struct ut_expected_io *expected_io;
int rc;
bdev_opts.opts_size = sizeof(bdev_opts);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 2;
bdev_opts.bdev_io_cache_size = 1;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
spdk_bdev_initialize(bdev_init_cb, NULL);
@ -2677,17 +2678,17 @@ bdev_io_alignment(void)
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc = NULL;
struct spdk_io_channel *io_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 20,
.bdev_io_cache_size = 2,
};
struct spdk_bdev_opts bdev_opts = {};
int rc;
void *buf = NULL;
struct iovec iovs[2];
int iovcnt;
uint64_t alignment;
bdev_opts.opts_size = sizeof(bdev_opts);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 20;
bdev_opts.bdev_io_cache_size = 2;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
spdk_bdev_initialize(bdev_init_cb, NULL);
@ -2898,16 +2899,17 @@ bdev_io_alignment_with_boundary(void)
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc = NULL;
struct spdk_io_channel *io_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 20,
.bdev_io_cache_size = 2,
};
struct spdk_bdev_opts bdev_opts = {};
int rc;
void *buf = NULL;
struct iovec iovs[2];
int iovcnt;
uint64_t alignment;
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 20;
bdev_opts.bdev_io_cache_size = 2;
bdev_opts.opts_size = sizeof(bdev_opts);
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
@ -4081,15 +4083,15 @@ bdev_io_abort(void)
struct spdk_io_channel *io_ch;
struct spdk_bdev_channel *channel;
struct spdk_bdev_mgmt_channel *mgmt_ch;
struct spdk_bdev_opts bdev_opts = {
.bdev_io_pool_size = 7,
.bdev_io_cache_size = 2,
};
struct spdk_bdev_opts bdev_opts = {};
struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2];
uint64_t io_ctx1 = 0, io_ctx2 = 0, i;
int rc;
bdev_opts.opts_size = sizeof(bdev_opts);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 7;
bdev_opts.bdev_io_cache_size = 2;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
spdk_bdev_initialize(bdev_init_cb, NULL);
@ -4263,6 +4265,43 @@ bdev_io_abort(void)
poll_threads();
}
static void
bdev_set_options_test(void)
{
struct spdk_bdev_opts bdev_opts = {};
int rc;
/* Case1: Do not set opts_size */
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == -1);
spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts));
bdev_opts.bdev_io_pool_size = 4;
bdev_opts.bdev_io_cache_size = 2;
bdev_opts.small_buf_pool_size = 4;
/* Case 2: Do not set valid small_buf_pool_size and large_buf_pool_size */
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == -1);
/* Case 3: Do not set valid large_buf_pool_size */
bdev_opts.small_buf_pool_size = BUF_SMALL_POOL_SIZE;
bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE - 1;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == -1);
/* Case4: set valid large buf_pool_size */
bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
/* Case5: Set different valid value for small and large buf pool */
bdev_opts.large_buf_pool_size = BUF_SMALL_POOL_SIZE + 3;
bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE + 3;
rc = spdk_bdev_set_opts(&bdev_opts);
CU_ASSERT(rc == 0);
}
int
main(int argc, char **argv)
{
@ -4302,6 +4341,7 @@ main(int argc, char **argv)
CU_ADD_TEST(suite, lock_lba_range_with_io_outstanding);
CU_ADD_TEST(suite, lock_lba_range_overlapped);
CU_ADD_TEST(suite, bdev_io_abort);
CU_ADD_TEST(suite, bdev_set_options_test);
allocate_cores(1);
allocate_threads(1);