From cc5e5b9e50325aa315df04b1133f26973bee9595 Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Wed, 14 Feb 2018 10:26:10 +0800 Subject: [PATCH] iscsi: Enable the sock fd group polling mechanism This patch is used to implement the sock fd event group polling mechanism if there are incoming data from network (read event in SPDK iSCSI target side) , thus we can awake many connections one time, and it can reduce the system call overhead. Change-Id: I76c26a89ef9365d7e1ccec616985e7435253896b Signed-off-by: Ziye Yang Reviewed-on: https://review.gerrithub.io/399796 Tested-by: SPDK Automated Test System Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris --- lib/iscsi/conn.c | 60 +++++++++++++++++++++------- lib/iscsi/iscsi.h | 1 + lib/iscsi/iscsi_subsystem.c | 14 +++++++ test/unit/lib/iscsi/conn.c/conn_ut.c | 13 ++++++ 4 files changed, 73 insertions(+), 15 deletions(-) diff --git a/lib/iscsi/conn.c b/lib/iscsi/conn.c index 2628526fb..ec50be06e 100644 --- a/lib/iscsi/conn.c +++ b/lib/iscsi/conn.c @@ -81,6 +81,8 @@ void spdk_iscsi_conn_full_feature_do_work(void *arg); static void spdk_iscsi_conn_full_feature_migrate(void *arg1, void *arg2); static void spdk_iscsi_conn_stop(struct spdk_iscsi_conn *conn); +static void spdk_iscsi_conn_sock_cb(void *arg, struct spdk_sock_group *group, + struct spdk_sock *sock); static struct spdk_iscsi_conn * allocate_conn(void) @@ -170,6 +172,30 @@ int spdk_initialize_iscsi_conns(void) return 0; } +static void +spdk_iscsi_poll_group_add_conn_sock(struct spdk_iscsi_conn *conn) +{ + struct spdk_iscsi_poll_group *poll_group = &g_spdk_iscsi.poll_group[spdk_env_get_current_core()]; + int rc; + + rc = spdk_sock_group_add_sock(poll_group->sock_group, conn->sock, spdk_iscsi_conn_sock_cb, conn); + if (rc < 0) { + SPDK_ERRLOG("Failed to add sock=%p of conn=%p\n", conn->sock, conn); + } +} + +static void +spdk_iscsi_poll_group_remove_conn_sock(struct spdk_iscsi_conn *conn) +{ + struct spdk_iscsi_poll_group *poll_group = &g_spdk_iscsi.poll_group[spdk_env_get_current_core()]; + int rc; + + rc = spdk_sock_group_remove_sock(poll_group->sock_group, conn->sock); + if (rc < 0) { + SPDK_ERRLOG("Failed to remove sock=%p of conn=%p\n", conn->sock, conn); + } +} + static void spdk_iscsi_poll_group_add_conn(struct spdk_iscsi_conn *conn, spdk_iscsi_conn_fn fn) @@ -178,6 +204,7 @@ spdk_iscsi_poll_group_add_conn(struct spdk_iscsi_conn *conn, conn->fn = fn; STAILQ_INSERT_TAIL(&poll_group->connections, conn, link); + spdk_iscsi_poll_group_add_conn_sock(conn); } static void @@ -500,6 +527,7 @@ void spdk_iscsi_conn_destruct(struct spdk_iscsi_conn *conn) } spdk_clear_all_transfer_task(conn, NULL); + spdk_iscsi_poll_group_remove_conn_sock(conn); spdk_sock_close(&conn->sock); spdk_poller_unregister(&conn->logout_timer); spdk_poller_unregister(&conn->flush_poller); @@ -1106,11 +1134,26 @@ spdk_iscsi_conn_handle_incoming_pdus(struct spdk_iscsi_conn *conn) return i; } +static void +spdk_iscsi_conn_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) +{ + struct spdk_iscsi_conn *conn = arg; + int rc; + + assert(conn != NULL); + + /* Handle incoming PDUs */ + rc = spdk_iscsi_conn_handle_incoming_pdus(conn); + if (rc < 0) { + conn->state = ISCSI_CONN_STATE_EXITING; + spdk_iscsi_conn_flush_pdus(conn); + } +} + static int spdk_iscsi_conn_execute(struct spdk_iscsi_conn *conn) { int rc = 0; - bool conn_active = false; if (conn->state == ISCSI_CONN_STATE_EXITED) { return -1; @@ -1126,22 +1169,8 @@ spdk_iscsi_conn_execute(struct spdk_iscsi_conn *conn) goto conn_exit; } - /* Handle incoming PDUs */ - rc = spdk_iscsi_conn_handle_incoming_pdus(conn); - if (rc < 0) { - conn->state = ISCSI_CONN_STATE_EXITING; - spdk_iscsi_conn_flush_pdus(conn); - goto conn_exit; - } else if (rc > 0) { - conn_active = true; - } - spdk_iscsi_conn_handle_queued_datain_tasks(conn); - if (conn_active) { - return 1; - } - conn_exit: if (conn->state == ISCSI_CONN_STATE_EXITING) { spdk_iscsi_conn_destruct(conn); @@ -1210,6 +1239,7 @@ spdk_iscsi_conn_login_do_work(void *arg) pthread_mutex_unlock(&target->mutex); } + spdk_iscsi_poll_group_remove_conn_sock(conn); spdk_iscsi_conn_stop(conn); __sync_fetch_and_add(&g_num_connections[lcore], 1); diff --git a/lib/iscsi/iscsi.h b/lib/iscsi/iscsi.h index c83a35afd..c3d8c1e91 100644 --- a/lib/iscsi/iscsi.h +++ b/lib/iscsi/iscsi.h @@ -260,6 +260,7 @@ struct spdk_iscsi_poll_group { uint32_t core; struct spdk_poller *poller; STAILQ_HEAD(connections, spdk_iscsi_conn) connections; + struct spdk_sock_group *sock_group; }; struct spdk_iscsi_globals { diff --git a/lib/iscsi/iscsi_subsystem.c b/lib/iscsi/iscsi_subsystem.c index 4902646fb..7dd980e59 100644 --- a/lib/iscsi/iscsi_subsystem.c +++ b/lib/iscsi/iscsi_subsystem.c @@ -35,6 +35,7 @@ #include "spdk/stdinc.h" #include "spdk/env.h" #include "spdk/string.h" +#include "spdk/sock.h" #include "iscsi/iscsi.h" #include "iscsi/init_grp.h" @@ -835,6 +836,14 @@ spdk_iscsi_poll_group_poll(void *ctx) { struct spdk_iscsi_poll_group *group = ctx; struct spdk_iscsi_conn *conn, *tmp; + int rc; + + if (!STAILQ_EMPTY(&group->connections)) { + rc = spdk_sock_group_poll(group->sock_group); + if (rc < 0) { + SPDK_ERRLOG("Failed to poll sock_group=%p\n", group->sock_group); + } + } STAILQ_FOREACH_SAFE(conn, &group->connections, link, tmp) { conn->fn(conn); @@ -857,6 +866,9 @@ iscsi_create_poll_group(void *ctx) assert(pg != NULL); STAILQ_INIT(&pg->connections); + pg->sock_group = spdk_sock_group_create(); + assert(pg->sock_group != NULL); + pg->poller = spdk_poller_register(spdk_iscsi_poll_group_poll, pg, 0); } @@ -868,7 +880,9 @@ iscsi_unregister_poll_group(void *ctx) pg = &g_spdk_iscsi.poll_group[spdk_env_get_current_core()]; assert(pg != NULL); assert(pg->poller != NULL); + assert(pg->sock_group != NULL); + spdk_sock_group_close(&pg->sock_group); spdk_poller_unregister(&pg->poller); } diff --git a/test/unit/lib/iscsi/conn.c/conn_ut.c b/test/unit/lib/iscsi/conn.c/conn_ut.c index 0174fed73..1588b11bc 100644 --- a/test/unit/lib/iscsi/conn.c/conn_ut.c +++ b/test/unit/lib/iscsi/conn.c/conn_ut.c @@ -139,6 +139,19 @@ spdk_sock_set_sendbuf(struct spdk_sock *sock, int sz) return 0; } +int +spdk_sock_group_add_sock(struct spdk_sock_group *group, struct spdk_sock *sock, + spdk_sock_cb cb_fn, void *cb_arg) +{ + return 0; +} + +int +spdk_sock_group_remove_sock(struct spdk_sock_group *group, struct spdk_sock *sock) +{ + return 0; +} + void spdk_scsi_task_put(struct spdk_scsi_task *task) {