ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bdev/compress: fix qpair assignment code

Browse Source 
Was previously creating the number of qp based on the # of cores
reported via DPDK function.  Also was assigning one qp per comp_bdev.
After testing w/multiple threads in fio 2 things needed to change:

* remove use of rte core related functions as fio threads are not EAL
threads.  This required choosing an upper limit for the number of
threads that the bdev module will support
* changing the qp assignment code such that comp_bdevs share the
same device_qp if on the same thread

Fixes issue #879

Change-Id: Ib968ee04bef63013616546341974a4431d8f266a
Signed-off-by: paul luse <paul.e.luse@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/463107
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
paul luse 2019-07-24 17:00:47 -04:00 committed by Jim Harris
parent 59339feac3
commit e9211cbf45
1 changed files with 39 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
lib/bdev/compress/vbdev_compress.c
View File

@ -76,11 +76,16 @@ struct compress_dev {
};
static TAILQ_HEAD(, compress_dev) g_compress_devs = TAILQ_HEAD_INITIALIZER(g_compress_devs);
/* Although ISAL PMD reports 'unlimited' qpairs, it has an unplanned limit of 99 due to
* the length of the internal ring name that it creates, it breaks a limit in the generic
* ring code and fails the qp initialization.
*/
#define MAX_NUM_QP 99
/* Global list and lock for unique device/queue pair combos */
struct comp_device_qp {
struct compress_dev *device; /* ptr to compression device */
uint8_t qp; /* queue pair for this node */
bool in_use; /* whether this node is in use or not */
struct spdk_thread *thread; /* thead that this qp is assigned to */
TAILQ_ENTRY(comp_device_qp) link;
};
static TAILQ_HEAD(, comp_device_qp) g_comp_device_qp = TAILQ_HEAD_INITIALIZER(g_comp_device_qp);
@ -186,10 +191,10 @@ shinfo_free_cb(void *arg1, void *arg2)
/* Called by vbdev_init_compress_drivers() to init each discovered compression device */
static int
create_compress_dev(uint8_t index, uint16_t num_lcores)
create_compress_dev(uint8_t index)
{
struct compress_dev *device;
uint16_t q_pairs = num_lcores;
uint16_t q_pairs;
uint8_t cdev_id;
int rc, i;
struct comp_device_qp *dev_qp;
@ -207,9 +212,9 @@ create_compress_dev(uint8_t index, uint16_t num_lcores)
/* Zero means no limit so choose number of lcores. */
if (device->cdev_info.max_nb_queue_pairs == 0) {
q_pairs = num_lcores;
q_pairs = MAX_NUM_QP;
} else {
q_pairs = spdk_min(device->cdev_info.max_nb_queue_pairs, num_lcores);
q_pairs = spdk_min(device->cdev_info.max_nb_queue_pairs, MAX_NUM_QP);
}
/* Configure the compression device. */
@ -233,10 +238,19 @@ create_compress_dev(uint8_t index, uint16_t num_lcores)
NUM_MAX_INFLIGHT_OPS,
rte_socket_id());
if (rc) {
SPDK_ERRLOG("Failed to setup queue pair on "
"compressdev %u\n", cdev_id);
rc = -EINVAL;
goto err;
if (i > 0) {
q_pairs = i;
SPDK_NOTICELOG("FYI failed to setup a queue pair on "
"compressdev %u with error %u "
"so limiting to %u qpairs\n",
cdev_id, rc, q_pairs);
break;
} else {
SPDK_ERRLOG("Failed to setup queue pair on "
"compressdev %u with error %u\n", cdev_id, rc);
rc = -EINVAL;
goto err;
}
}
}
@ -277,7 +291,7 @@ create_compress_dev(uint8_t index, uint16_t num_lcores)
}
dev_qp->device = device;
dev_qp->qp = i;
dev_qp->in_use = false;
dev_qp->thread = NULL;
TAILQ_INSERT_TAIL(&g_comp_device_qp, dev_qp, link);
}
@ -306,7 +320,6 @@ static int
vbdev_init_compress_drivers(void)
{
uint8_t cdev_count, i;
uint16_t num_lcores = rte_lcore_count();
struct compress_dev *tmp_dev;
struct compress_dev *device;
int rc;
@ -350,7 +363,7 @@ vbdev_init_compress_drivers(void)
/* Init all devices */
for (i = 0; i < cdev_count; i++) {
rc = create_compress_dev(i, num_lcores);
rc = create_compress_dev(i);
if (rc != 0) {
goto error_create_compress_devs;
}
@ -1198,11 +1211,16 @@ comp_bdev_ch_create_cb(void *io_device, void *ctx_buf)
/* Now assign a q pair */
pthread_mutex_lock(&g_comp_device_qp_lock);
TAILQ_FOREACH(device_qp, &g_comp_device_qp, link) {
if ((strcmp(device_qp->device->cdev_info.driver_name, comp_bdev->drv_name) == 0) &&
(device_qp->in_use == false)) {
comp_bdev->device_qp = device_qp;
device_qp->in_use = true;
break;
if ((strcmp(device_qp->device->cdev_info.driver_name, comp_bdev->drv_name) == 0)) {
if (device_qp->thread == spdk_get_thread()) {
comp_bdev->device_qp = device_qp;
break;
}
if (device_qp->thread == NULL) {
comp_bdev->device_qp = device_qp;
device_qp->thread = spdk_get_thread();
break;
}
}
}
pthread_mutex_unlock(&g_comp_device_qp_lock);
@ -1217,10 +1235,10 @@ comp_bdev_ch_create_cb(void *io_device, void *ctx_buf)
static void
_clear_qp_and_put_channel(struct vbdev_compress *comp_bdev)
{
pthread_mutex_lock(&g_comp_device_qp_lock);
comp_bdev->device_qp->in_use = false;
pthread_mutex_unlock(&g_comp_device_qp_lock);
/* Note: comp_bdevs can share a device_qp if they are
* on the same thread so we leave the device_qp element
* alone for this comp_bdev and just clear the reduce thread.
*/
spdk_put_io_channel(comp_bdev->base_ch);
comp_bdev->reduce_thread = NULL;
spdk_poller_unregister(&comp_bdev->poller);