ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QoS/Bdev: add the Read/Write separate bandwidth rate limits

Browse Source 
This patch adds the support of read and write separate
bandwidth rate limits control with the configuration file.

Below is the example (in MiB) for the configuration section:

[QoS]
  Limit_Read_BPS Malloc0 100
  Limit_Write_BPS Nvme0n1 200

Change-Id: I0221516ce70c3fbb07b9e80c1c814ed5ba271c88
Signed-off-by: GangCao <gang.cao@intel.com>
Reviewed-on: https://review.gerrithub.io/c/416672
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
GangCao 2018-06-21 22:15:02 -04:00 committed by Ben Walker
parent 1da6e2f5c5
commit 316cb3b150
6 changed files with 137 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/spdk/bdev.h
View File

@ -112,6 +112,10 @@ enum spdk_bdev_qos_rate_limit_type {
SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT = 0,
/** Byte per second rate limit for both read and write */
SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT,
/** Byte per second rate limit for read only */
SPDK_BDEV_QOS_R_BPS_RATE_LIMIT,
/** Byte per second rate limit for write only */
SPDK_BDEV_QOS_W_BPS_RATE_LIMIT,
/** Keep last */
SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES
};

77
lib/bdev/bdev.c
View File

@ -81,8 +81,12 @@ int __itt_init_ittlib(const char *, __itt_group_id);
#define SPDK_BDEV_POOL_ALIGNMENT 512
static const char *qos_conf_type[] = {"Limit_IOPS", "Limit_BPS"};
static const char *qos_rpc_type[] = {"rw_ios_per_sec", "rw_mbytes_per_sec"};
static const char *qos_conf_type[] = {"Limit_IOPS",
"Limit_BPS", "Limit_Read_BPS", "Limit_Write_BPS"
};
static const char *qos_rpc_type[] = {"rw_ios_per_sec",
"rw_mbytes_per_sec", "r_mbytes_per_sec", "w_mbytes_per_sec"
};
TAILQ_HEAD(spdk_bdev_list, spdk_bdev);
@ -1214,6 +1218,8 @@ _spdk_bdev_qos_is_iops_rate_limit(enum spdk_bdev_qos_rate_limit_type limit)
case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:
return true;
case SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT:
case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:
case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:
return false;
case SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES:
default:
@ -1237,6 +1243,25 @@ _spdk_bdev_qos_io_to_limit(struct spdk_bdev_io *bdev_io)
}
}
static bool
_spdk_bdev_is_read_io(struct spdk_bdev_io *bdev_io)
{
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_NVME_IO:
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
/* Bit 1 (0x2) set for read operation */
if (bdev_io->u.nvme_passthru.cmd.opc & SPDK_NVME_OPC_READ) {
return true;
} else {
return false;
}
case SPDK_BDEV_IO_TYPE_READ:
return true;
default:
return false;
}
}
static uint64_t
_spdk_bdev_get_io_size_in_byte(struct spdk_bdev_io *bdev_io)
{
@ -1266,6 +1291,26 @@ _spdk_bdev_qos_rw_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_
}
}
static bool
_spdk_bdev_qos_r_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (_spdk_bdev_is_read_io(io) == false) {
return false;
}
return _spdk_bdev_qos_rw_queue_io(limit, io);
}
static bool
_spdk_bdev_qos_w_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (_spdk_bdev_is_read_io(io) == true) {
return false;
}
return _spdk_bdev_qos_rw_queue_io(limit, io);
}
static void
_spdk_bdev_qos_rw_iops_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
@ -1278,6 +1323,26 @@ _spdk_bdev_qos_rw_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spd
limit->remaining_this_timeslice -= _spdk_bdev_get_io_size_in_byte(io);
}
static void
_spdk_bdev_qos_r_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (_spdk_bdev_is_read_io(io) == false) {
return;
}
return _spdk_bdev_qos_rw_bps_update_quota(limit, io);
}
static void
_spdk_bdev_qos_w_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (_spdk_bdev_is_read_io(io) == true) {
return;
}
return _spdk_bdev_qos_rw_bps_update_quota(limit, io);
}
static void
_spdk_bdev_qos_set_ops(struct spdk_bdev_qos *qos)
{
@ -1299,6 +1364,14 @@ _spdk_bdev_qos_set_ops(struct spdk_bdev_qos *qos)
qos->rate_limits[i].queue_io = _spdk_bdev_qos_rw_queue_io;
qos->rate_limits[i].update_quota = _spdk_bdev_qos_rw_bps_update_quota;
break;
case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:
qos->rate_limits[i].queue_io = _spdk_bdev_qos_r_queue_io;
qos->rate_limits[i].update_quota = _spdk_bdev_qos_r_bps_update_quota;
break;
case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:
qos->rate_limits[i].queue_io = _spdk_bdev_qos_w_queue_io;
qos->rate_limits[i].update_quota = _spdk_bdev_qos_w_bps_update_quota;
break;
default:
break;
}

8
lib/bdev/rpc/bdev_rpc.c
View File

@ -550,9 +550,8 @@ static void
spdk_rpc_set_bdev_qos_limit(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_set_bdev_qos_limit req = {NULL, {UINT64_MAX, UINT64_MAX}};
struct rpc_set_bdev_qos_limit req = {NULL, {UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX}};
struct spdk_bdev *bdev;
bool valid_limit = false;
int i;
if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_decoders,
@ -572,11 +571,10 @@ spdk_rpc_set_bdev_qos_limit(struct spdk_jsonrpc_request *request,
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (req.limits[i] != UINT64_MAX) {
valid_limit = true;
break;
}
}
if (valid_limit == false) {
if (i == SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES) {
SPDK_ERRLOG("no rate limits specified\n");
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
"No rate limits specified");

2
test/bdev/bdev.conf.in
View File

@ -35,6 +35,8 @@
# Assign 100 (MB) bandwidth for the Malloc3 block
# device
Limit_BPS Malloc3 100
# Assign 50 (MB) read only bandwidth for the AIO0 block device
Limit_Read_BPS AIO0 50
[RAID0]
Name raid0

4
test/spdkcli/match_files/spdkcli_details_vhost.test.match
View File

@ -1,8 +1,10 @@
{
"aliases": [],
"assigned_rate_limits": {
"r_mbytes_per_sec": $(N),
"rw_ios_per_sec": $(N),
"rw_mbytes_per_sec": $(N)
"rw_mbytes_per_sec": $(N),
"w_mbytes_per_sec": $(N)
},
"block_size": $(N),
"claimed": false,

71
test/unit/lib/bdev/mt/bdev.c/bdev_ut.c
View File

@ -617,13 +617,16 @@ basic_qos(void)
SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL);
TAILQ_INIT(&bdev->internal.qos->queued);
/*
* Enable both IOPS and bandwidth rate limits.
* In this case, both rate limits will take equal effect.
* Enable read/write IOPS, read only byte per second and
* read/write byte per second rate limits.
* In this case, all rate limits will take equal effect.
*/
/* 2000 I/O per second, or 2 per millisecond */
/* 2000 read/write I/O per second, or 2 per millisecond */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000;
/* 8K byte per millisecond with 4K block size */
/* 8K read/write byte per millisecond with 4K block size */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000;
/* 8K read only byte per millisecond with 4K block size */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 8192000;
g_get_io_channel = true;
@ -715,7 +718,7 @@ io_during_qos_queue(void)
struct spdk_io_channel *io_ch[2];
struct spdk_bdev_channel *bdev_ch[2];
struct spdk_bdev *bdev;
enum spdk_bdev_io_status status0, status1;
enum spdk_bdev_io_status status0, status1, status2;
int rc;
setup_test();
@ -727,13 +730,19 @@ io_during_qos_queue(void)
SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL);
TAILQ_INIT(&bdev->internal.qos->queued);
/*
* Enable both IOPS and bandwidth rate limits.
* In this case, IOPS rate limit will take effect first.
* Enable read/write IOPS, read only byte per sec, write only
* byte per sec and read/write byte per sec rate limits.
* In this case, both read only and write only byte per sec
* rate limit will take effect.
*/
/* 1000 I/O per second, or 1 per millisecond */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 1000;
/* 4000 read/write I/O per second, or 4 per millisecond */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 4000;
/* 8K byte per millisecond with 4K block size */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000;
/* 4K byte per millisecond with 4K block size */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 4096000;
/* 4K byte per millisecond with 4K block size */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 4096000;
g_get_io_channel = true;
@ -748,7 +757,7 @@ io_during_qos_queue(void)
bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]);
CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED);
/* Send two I/O */
/* Send two read I/Os */
status1 = SPDK_BDEV_IO_STATUS_PENDING;
rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1);
CU_ASSERT(rc == 0);
@ -758,6 +767,11 @@ io_during_qos_queue(void)
rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0);
CU_ASSERT(rc == 0);
CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING);
/* Send one write I/O */
status2 = SPDK_BDEV_IO_STATUS_PENDING;
rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status2);
CU_ASSERT(rc == 0);
CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_PENDING);
/* Complete any I/O that arrived at the disk */
poll_threads();
@ -767,12 +781,14 @@ io_during_qos_queue(void)
stub_complete_io(g_bdev.io_target, 0);
poll_threads();
/* Only one of the I/O should complete. (logical XOR) */
/* Only one of the two read I/Os should complete. (logical XOR) */
if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) {
CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING);
} else {
CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS);
}
/* The write I/O should complete. */
CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS);
/* Advance in time by a millisecond */
spdk_delay_us(1000);
@ -785,7 +801,7 @@ io_during_qos_queue(void)
stub_complete_io(g_bdev.io_target, 0);
poll_threads();
/* Now the second I/O should be done */
/* Now the second read I/O should be done */
CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS);
CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS);
@ -817,13 +833,17 @@ io_during_qos_reset(void)
SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL);
TAILQ_INIT(&bdev->internal.qos->queued);
/*
* Enable both IOPS and bandwidth rate limits.
* In this case, bandwidth rate limit will take effect first.
* Enable read/write IOPS, write only byte per sec and
* read/write byte per second rate limits.
* In this case, read/write byte per second rate limit will
* take effect first.
*/
/* 2000 I/O per second, or 2 per millisecond */
/* 2000 read/write I/O per second, or 2 per millisecond */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000;
/* 4K byte per millisecond with 4K block size */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000;
/* 8K byte per millisecond with 4K block size */
bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 8192000;
g_get_io_channel = true;
@ -840,11 +860,11 @@ io_during_qos_reset(void)
/* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */
status1 = SPDK_BDEV_IO_STATUS_PENDING;
rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1);
rc = spdk_bdev_write_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1);
CU_ASSERT(rc == 0);
set_thread(0);
status0 = SPDK_BDEV_IO_STATUS_PENDING;
rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0);
rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0);
CU_ASSERT(rc == 0);
poll_threads();
@ -1181,12 +1201,16 @@ qos_dynamic_enable(void)
set_thread(0);
/*
* Enable QoS: IOPS and byte per second rate limits.
* Enable QoS: Read/Write IOPS, Read/Write byte,
* Read only byte and Write only byte per second
* rate limits.
* More than 10 I/Os allowed per timeslice.
*/
status = -1;
limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000;
limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100;
limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 100;
limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 10;
spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status);
poll_threads();
CU_ASSERT(status == 0);
@ -1225,18 +1249,23 @@ qos_dynamic_enable(void)
CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING);
poll_threads();
/* Disable QoS: IOPS rate limit */
/*
* Disable QoS: Read/Write IOPS, Read/Write byte,
* Read only byte rate limits
*/
status = -1;
limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0;
limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0;
limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 0;
spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status);
poll_threads();
CU_ASSERT(status == 0);
CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0);
CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0);
/* Disable QoS: Byte per second rate limit */
/* Disable QoS: Write only Byte per second rate limit */
status = -1;
limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0;
limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 0;
spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status);
poll_threads();
CU_ASSERT(status == 0);