ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

iscsi: Use spdk_mempool for session and pdu related pool

Browse Source 
Purpose: To remove the dpdk dependency for iSCSI.

Change-Id: I01ef910496d2029ba755dd6d0c06d61c248b8dfe
Signed-off-by: Ziye Yang <optimistyzy@gmail.com>
Reviewed-on: https://review.gerrithub.io/390938
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: <shuhei.matsumoto.xt@hitachi.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Ziye Yang 2017-12-08 16:12:30 +08:00 committed by Jim Harris
parent f566060ab4
commit e78188042c
3 changed files with 44 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/iscsi/iscsi.c
View File

@ -34,9 +34,6 @@
#include "spdk/stdinc.h" #include "spdk/stdinc.h"
#include <rte_config.h>
#include <rte_mempool.h>
#include "spdk/crc32.h" #include "spdk/crc32.h"
#include "spdk/endian.h" #include "spdk/endian.h"
#include "spdk/env.h" #include "spdk/env.h"
@ -356,7 +353,7 @@ int
spdk_iscsi_read_pdu(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu **_pdu) spdk_iscsi_read_pdu(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu **_pdu)
{ {
struct spdk_iscsi_pdu *pdu; struct spdk_iscsi_pdu *pdu;
struct rte_mempool *pool; struct spdk_mempool *pool;
uint32_t crc32c; uint32_t crc32c;
int ahs_len; int ahs_len;
int data_len; int data_len;
@ -444,7 +441,7 @@ spdk_iscsi_read_pdu(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu **_pdu)
conn->pdu_in_progress = NULL; conn->pdu_in_progress = NULL;
return SPDK_ISCSI_CONNECTION_FATAL; return SPDK_ISCSI_CONNECTION_FATAL;
} }
rte_mempool_get(pool, (void **)&pdu->mobj); pdu->mobj = spdk_mempool_get(pool);
if (pdu->mobj == NULL) { if (pdu->mobj == NULL) {
*_pdu = NULL; *_pdu = NULL;
return SPDK_SUCCESS; return SPDK_SUCCESS;
@ -4525,7 +4522,7 @@ void spdk_free_sess(struct spdk_iscsi_sess *sess)
spdk_iscsi_param_free(sess->params); spdk_iscsi_param_free(sess->params);
free(sess->conns); free(sess->conns);
spdk_scsi_port_free(&sess->initiator_port); spdk_scsi_port_free(&sess->initiator_port);
rte_mempool_put(g_spdk_iscsi.session_pool, (void *)sess); spdk_mempool_put(g_spdk_iscsi.session_pool, (void *)sess);
} }
static int static int
@ -4536,8 +4533,8 @@ spdk_create_iscsi_sess(struct spdk_iscsi_conn *conn,
struct spdk_iscsi_sess *sess; struct spdk_iscsi_sess *sess;
int rc; int rc;
rc = rte_mempool_get(g_spdk_iscsi.session_pool, (void **)&sess); sess = spdk_mempool_get(g_spdk_iscsi.session_pool);
if ((rc < 0) || !sess) { if (!sess) {
SPDK_ERRLOG("Unable to get session object\n"); SPDK_ERRLOG("Unable to get session object\n");
SPDK_ERRLOG("MaxSessions set to %d\n", g_spdk_iscsi.MaxSessions); SPDK_ERRLOG("MaxSessions set to %d\n", g_spdk_iscsi.MaxSessions);
return -ENOMEM; return -ENOMEM;

10
lib/iscsi/iscsi.h
View File

@ -148,7 +148,7 @@
#define ISCSI_AHS_LEN 60 #define ISCSI_AHS_LEN 60
struct spdk_mobj { struct spdk_mobj {
struct rte_mempool *mp; struct spdk_mempool *mp;
void *buf; void *buf;
size_t len; size_t len;
uint64_t reserved; /* do not use */ uint64_t reserved; /* do not use */
@ -287,10 +287,10 @@ struct spdk_iscsi_globals {
uint32_t ErrorRecoveryLevel; uint32_t ErrorRecoveryLevel;
uint32_t AllowDuplicateIsid; uint32_t AllowDuplicateIsid;
struct rte_mempool *pdu_pool; struct spdk_mempool *pdu_pool;
struct rte_mempool *pdu_immediate_data_pool; struct spdk_mempool *pdu_immediate_data_pool;
struct rte_mempool *pdu_data_out_pool; struct spdk_mempool *pdu_data_out_pool;
struct rte_mempool *session_pool; struct spdk_mempool *session_pool;
struct spdk_mempool *task_pool; struct spdk_mempool *task_pool;
struct spdk_iscsi_sess **session; struct spdk_iscsi_sess **session;

97
lib/iscsi/iscsi_subsystem.c
View File

@ -33,10 +33,7 @@
*/ */
#include "spdk/stdinc.h" #include "spdk/stdinc.h"
#include "spdk/env.h"
#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_version.h>
#include "iscsi/iscsi.h" #include "iscsi/iscsi.h"
#include "iscsi/init_grp.h" #include "iscsi/init_grp.h"
@ -44,8 +41,6 @@
#include "iscsi/conn.h" #include "iscsi/conn.h"
#include "iscsi/task.h" #include "iscsi/task.h"
#include "spdk/env.h"
#include "spdk_internal/event.h" #include "spdk_internal/event.h"
#include "spdk_internal/log.h" #include "spdk_internal/log.h"
@ -303,7 +298,7 @@ spdk_iscsi_config_dump_target_nodes(FILE *fp)
} }
static void static void
spdk_mobj_ctor(struct rte_mempool *mp, __attribute__((unused)) void *arg, spdk_mobj_ctor(struct spdk_mempool *mp, __attribute__((unused)) void *arg,
void *_m, __attribute__((unused)) unsigned i) void *_m, __attribute__((unused)) unsigned i)
{ {
struct spdk_mobj *m = _m; struct spdk_mobj *m = _m;
@ -320,12 +315,7 @@ spdk_mobj_ctor(struct rte_mempool *mp, __attribute__((unused)) void *arg,
* right before the 512B aligned buffer area. * right before the 512B aligned buffer area.
*/ */
phys_addr = (uint64_t *)m->buf - 1; phys_addr = (uint64_t *)m->buf - 1;
*phys_addr = spdk_vtophys(m) + off;
#if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
*phys_addr = rte_mempool_virt2iova(m) + off;
#else
*phys_addr = rte_mempool_virt2phy(mp, m) + off;
#endif
} }
#define NUM_PDU_PER_CONNECTION(iscsi) (2 * (iscsi->MaxQueueDepth + MAX_LARGE_DATAIN_PER_CONNECTION + 8)) #define NUM_PDU_PER_CONNECTION(iscsi) (2 * (iscsi->MaxQueueDepth + MAX_LARGE_DATAIN_PER_CONNECTION + 8))
@ -342,35 +332,30 @@ static int spdk_iscsi_initialize_pdu_pool(void)
sizeof(struct spdk_mobj) + 512; sizeof(struct spdk_mobj) + 512;
/* create PDU pool */ /* create PDU pool */
iscsi->pdu_pool = rte_mempool_create("PDU_Pool", iscsi->pdu_pool = spdk_mempool_create("PDU_Pool",
PDU_POOL_SIZE(iscsi), PDU_POOL_SIZE(iscsi),
sizeof(struct spdk_iscsi_pdu), sizeof(struct spdk_iscsi_pdu),
256, 0, 256, SPDK_ENV_SOCKET_ID_ANY);
NULL, NULL, NULL, NULL,
SOCKET_ID_ANY, 0);
if (!iscsi->pdu_pool) { if (!iscsi->pdu_pool) {
SPDK_ERRLOG("create PDU pool failed\n"); SPDK_ERRLOG("create PDU pool failed\n");
return -1; return -1;
} }
iscsi->pdu_immediate_data_pool = iscsi->pdu_immediate_data_pool = spdk_mempool_create_ctor("PDU_immediate_data_Pool",
rte_mempool_create("PDU_immediate_data_Pool", IMMEDIATE_DATA_POOL_SIZE(iscsi),
IMMEDIATE_DATA_POOL_SIZE(iscsi), imm_mobj_size, 0,
imm_mobj_size, spdk_env_get_socket_id(spdk_env_get_current_core()),
0, 0, NULL, NULL, spdk_mobj_ctor, NULL);
spdk_mobj_ctor, NULL,
rte_socket_id(), 0);
if (!iscsi->pdu_immediate_data_pool) { if (!iscsi->pdu_immediate_data_pool) {
SPDK_ERRLOG("create PDU 8k pool failed\n"); SPDK_ERRLOG("create PDU 8k pool failed\n");
return -1; return -1;
} }
iscsi->pdu_data_out_pool = rte_mempool_create("PDU_data_out_Pool", iscsi->pdu_data_out_pool = spdk_mempool_create_ctor("PDU_data_out_Pool",
DATA_OUT_POOL_SIZE(iscsi), DATA_OUT_POOL_SIZE(iscsi),
dout_mobj_size, dout_mobj_size, 0,
0, 0, NULL, NULL, spdk_env_get_socket_id(spdk_env_get_current_core()),
spdk_mobj_ctor, NULL, spdk_mobj_ctor, NULL);
rte_socket_id(), 0);
if (!iscsi->pdu_data_out_pool) { if (!iscsi->pdu_data_out_pool) {
SPDK_ERRLOG("create PDU 64k pool failed\n"); SPDK_ERRLOG("create PDU 64k pool failed\n");
return -1; return -1;
@ -379,7 +364,7 @@ static int spdk_iscsi_initialize_pdu_pool(void)
return 0; return 0;
} }
static void spdk_iscsi_sess_ctor(struct rte_mempool *pool, void *arg, static void spdk_iscsi_sess_ctor(struct spdk_mempool *pool, void *arg,
void *session_buf, unsigned index) void *session_buf, unsigned index)
{ {
struct spdk_iscsi_globals *iscsi = arg; struct spdk_iscsi_globals *iscsi = arg;
@ -402,7 +387,7 @@ spdk_iscsi_initialize_task_pool(void)
iscsi->task_pool = spdk_mempool_create("SCSI_TASK_Pool", iscsi->task_pool = spdk_mempool_create("SCSI_TASK_Pool",
DEFAULT_TASK_POOL_SIZE, DEFAULT_TASK_POOL_SIZE,
sizeof(struct spdk_iscsi_task), sizeof(struct spdk_iscsi_task),
128, SOCKET_ID_ANY); 128, SPDK_ENV_SOCKET_ID_ANY);
if (!iscsi->task_pool) { if (!iscsi->task_pool) {
SPDK_ERRLOG("create task pool failed\n"); SPDK_ERRLOG("create task pool failed\n");
return -1; return -1;
@ -416,13 +401,11 @@ static int spdk_iscsi_initialize_session_pool(void)
{ {
struct spdk_iscsi_globals *iscsi = &g_spdk_iscsi; struct spdk_iscsi_globals *iscsi = &g_spdk_iscsi;
iscsi->session_pool = rte_mempool_create("Session_Pool", iscsi->session_pool = spdk_mempool_create_ctor("Session_Pool",
SESSION_POOL_SIZE(iscsi), SESSION_POOL_SIZE(iscsi),
sizeof(struct spdk_iscsi_sess), sizeof(struct spdk_iscsi_sess), 0,
0, 0, SPDK_ENV_SOCKET_ID_ANY,
NULL, NULL, spdk_iscsi_sess_ctor, iscsi);
spdk_iscsi_sess_ctor, iscsi,
SOCKET_ID_ANY, 0);
if (!iscsi->session_pool) { if (!iscsi->session_pool) {
SPDK_ERRLOG("create session pool failed\n"); SPDK_ERRLOG("create session pool failed\n");
return -1; return -1;
@ -449,23 +432,12 @@ spdk_iscsi_initialize_all_pools(void)
return 0; return 0;
} }
/*
* Wrapper to provide rte_mempool_avail_count() on older DPDK versions.
* Drop this if the minimum DPDK version is raised to at least 16.07.
*/
#if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1)
static unsigned rte_mempool_avail_count(const struct rte_mempool *pool)
{
return rte_mempool_count(pool);
}
#endif
static void static void
spdk_iscsi_check_pool(struct rte_mempool *pool, uint32_t count) spdk_iscsi_check_pool(struct spdk_mempool *pool, size_t count)
{ {
if (rte_mempool_avail_count(pool) != count) { if (spdk_mempool_count(pool) != count) {
SPDK_ERRLOG("rte_mempool_avail_count(%s) == %d, should be %d\n", SPDK_ERRLOG("spdk_mempool_count(%s) == %zu, should be %zu\n",
pool->name, rte_mempool_avail_count(pool), count); spdk_mempool_get_name(pool), spdk_mempool_count(pool), count);
} }
} }
@ -486,10 +458,10 @@ spdk_iscsi_free_pools(void)
{ {
struct spdk_iscsi_globals *iscsi = &g_spdk_iscsi; struct spdk_iscsi_globals *iscsi = &g_spdk_iscsi;
rte_mempool_free(iscsi->pdu_pool); spdk_mempool_free(iscsi->pdu_pool);
rte_mempool_free(iscsi->session_pool); spdk_mempool_free(iscsi->session_pool);
rte_mempool_free(iscsi->pdu_immediate_data_pool); spdk_mempool_free(iscsi->pdu_immediate_data_pool);
rte_mempool_free(iscsi->pdu_data_out_pool); spdk_mempool_free(iscsi->pdu_data_out_pool);
spdk_mempool_free(iscsi->task_pool); spdk_mempool_free(iscsi->task_pool);
} }
@ -508,24 +480,23 @@ void spdk_put_pdu(struct spdk_iscsi_pdu *pdu)
if (pdu->ref == 0) { if (pdu->ref == 0) {
if (pdu->mobj) { if (pdu->mobj) {
rte_mempool_put(pdu->mobj->mp, (void *)pdu->mobj); spdk_mempool_put(pdu->mobj->mp, (void *)pdu->mobj);
} }
if (pdu->data && !pdu->data_from_mempool) { if (pdu->data && !pdu->data_from_mempool) {
free(pdu->data); free(pdu->data);
} }
rte_mempool_put(g_spdk_iscsi.pdu_pool, (void *)pdu); spdk_mempool_put(g_spdk_iscsi.pdu_pool, (void *)pdu);
} }
} }
struct spdk_iscsi_pdu *spdk_get_pdu(void) struct spdk_iscsi_pdu *spdk_get_pdu(void)
{ {
struct spdk_iscsi_pdu *pdu; struct spdk_iscsi_pdu *pdu;
int rc;
rc = rte_mempool_get(g_spdk_iscsi.pdu_pool, (void **)&pdu); pdu = spdk_mempool_get(g_spdk_iscsi.pdu_pool);
if ((rc < 0) || !pdu) { if (!pdu) {
SPDK_ERRLOG("Unable to get PDU\n"); SPDK_ERRLOG("Unable to get PDU\n");
abort(); abort();
} }