From 2019ebf43e463887f58d56f45ba8bb99870c5dd4 Mon Sep 17 00:00:00 2001 From: Jan Kryl Date: Tue, 25 Jul 2017 20:02:41 +0200 Subject: [PATCH] Make iscsi app to work on freebsd Change-Id: Ic4221e044eb5af92aa06e468eb989bee285022af Signed-off-by: Jan Kryl Reviewed-on: https://review.gerrithub.io/371303 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Ben Walker --- app/Makefile | 2 +- app/iscsi_top/iscsi_top.cpp | 2 +- lib/Makefile | 3 +- lib/iscsi/conn.c | 128 +++++++++++++++++++++++++++++++++--- 4 files changed, 122 insertions(+), 13 deletions(-) diff --git a/app/Makefile b/app/Makefile index 2b4773cbf..2dcc54068 100644 --- a/app/Makefile +++ b/app/Makefile @@ -37,8 +37,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk DIRS-y += trace DIRS-y += nvmf_tgt DIRS-y += iscsi_top -ifeq ($(OS),Linux) DIRS-y += iscsi_tgt +ifeq ($(OS),Linux) DIRS-$(CONFIG_VHOST) += vhost endif diff --git a/app/iscsi_top/iscsi_top.cpp b/app/iscsi_top/iscsi_top.cpp index e6f529d62..f8a06308f 100644 --- a/app/iscsi_top/iscsi_top.cpp +++ b/app/iscsi_top/iscsi_top.cpp @@ -79,7 +79,7 @@ print_connections(void) int fd, i; char shm_name[64]; - snprintf(shm_name, sizeof(shm_name), "spdk_iscsi_conns.%d", g_shm_id); + snprintf(shm_name, sizeof(shm_name), "/spdk_iscsi_conns.%d", g_shm_id); fd = shm_open(shm_name, O_RDONLY, 0600); if (fd < 0) { fprintf(stderr, "Cannot open shared memory: %s\n", shm_name); diff --git a/lib/Makefile b/lib/Makefile index 56631dbdc..ce62f0d30 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -36,9 +36,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk DIRS-y += bdev blob blobfs conf copy cunit event json jsonrpc \ log env_dpdk net rpc trace util nvme nvmf scsi ioat \ - ut_mock + ut_mock iscsi ifeq ($(OS),Linux) -DIRS-y += iscsi DIRS-$(CONFIG_VHOST) += vhost endif diff --git a/lib/iscsi/conn.c b/lib/iscsi/conn.c index b668e0be7..cdf31a6b1 100644 --- a/lib/iscsi/conn.c +++ b/lib/iscsi/conn.c @@ -37,7 +37,11 @@ #include #include +#if defined(__FreeBSD__) +#include +#else #include +#endif #include "spdk/endian.h" #include "spdk/env.h" @@ -77,7 +81,7 @@ static uint32_t spdk_iscsi_conn_allocate_reactor(uint64_t cpumask); static void __add_idle_conn(void *arg1, void *arg2); /** Global variables used for managing idle connections. */ -static int g_epoll_fd = 0; +static int g_poll_fd = 0; static struct spdk_poller *g_idle_conn_poller; static STAILQ_HEAD(idle_list, spdk_iscsi_conn) g_idle_conn_list_head; @@ -133,12 +137,116 @@ spdk_find_iscsi_connection_by_id(int cid) } } +#if defined(__FreeBSD__) + static int init_idle_conns(void) { - assert(g_epoll_fd == 0); - g_epoll_fd = epoll_create1(0); - if (g_epoll_fd < 0) { + assert(g_poll_fd == 0); + g_poll_fd = kqueue(); + if (g_poll_fd < 0) { + SPDK_ERRLOG("kqueue failed master lcore\n"); + return -1; + } + + return 0; +} + +static int +add_idle_conn(struct spdk_iscsi_conn *conn) +{ + struct kevent event; + struct timespec ts = {0}; + int rc; + + EV_SET(&event, conn->sock, EVFILT_READ, EV_ADD, 0, 0, conn); + + rc = kevent(g_poll_fd, &event, 1, NULL, 0, &ts); + if (rc == -1) { + SPDK_ERRLOG("kevent(EV_ADD) failed\n"); + return -1; + } + + return 0; +} + +static int +del_idle_conn(struct spdk_iscsi_conn *conn) +{ + struct kevent event; + struct timespec ts = {0}; + char buf[64]; + int rc; + + EV_SET(&event, conn->sock, EVFILT_READ, EV_DELETE, 0, 0, NULL); + + rc = kevent(g_poll_fd, &event, 1, NULL, 0, &ts); + if (rc == -1) { + SPDK_ERRLOG("kevent(EV_DELETE) failed\n"); + return -1; + } + if (event.flags & EV_ERROR) { + strerror_r(event.data, buf, sizeof(buf)); + SPDK_ERRLOG("kevent(EV_DELETE) failed: %s\n", buf); + return -1; + } + + return 0; +} + +static void +check_idle_conns(void) +{ + struct kevent events[SPDK_MAX_POLLERS_PER_CORE]; + int i; + int nfds; + struct spdk_iscsi_conn *conn; + struct timespec ts = {0}; + + /* if nothing idle, can exit now */ + if (STAILQ_EMPTY(&g_idle_conn_list_head)) { + /* this kevent is needed to finish socket closing process */ + kevent(g_poll_fd, NULL, 0, events, SPDK_MAX_POLLERS_PER_CORE, &ts); + } + + /* Perform a non-blocking poll */ + nfds = kevent(g_poll_fd, NULL, 0, events, SPDK_MAX_POLLERS_PER_CORE, &ts); + if (nfds < 0) { + SPDK_ERRLOG("kevent failed! (ret: %d)\n", nfds); + return; + } + + if (nfds > SPDK_MAX_POLLERS_PER_CORE) { + SPDK_ERRLOG("kevent events exceeded limit! %d > %d\n", nfds, + SPDK_MAX_POLLERS_PER_CORE); + assert(0); + } + + /* + * In the case of any event cause (EPOLLIN or EPOLLERR) + * just make the connection active for normal process loop. + */ + for (i = 0; i < nfds; i++) { + + conn = (struct spdk_iscsi_conn *)events[i].udata; + + /* + * Flag the connection that an event was noticed + * such that during the list scan process it will + * be re-inserted into the active ring + */ + conn->pending_activate_event = true; + } +} + +#else + +static int +init_idle_conns(void) +{ + assert(g_poll_fd == 0); + g_poll_fd = epoll_create1(0); + if (g_poll_fd < 0) { SPDK_ERRLOG("epoll_create1 failed master lcore\n"); return -1; } @@ -156,7 +264,7 @@ add_idle_conn(struct spdk_iscsi_conn *conn) event.data.u64 = 0LL; event.data.ptr = conn; - rc = epoll_ctl(g_epoll_fd, EPOLL_CTL_ADD, conn->sock, &event); + rc = epoll_ctl(g_poll_fd, EPOLL_CTL_ADD, conn->sock, &event); if (rc == 0) { return 0; } else { @@ -175,7 +283,7 @@ del_idle_conn(struct spdk_iscsi_conn *conn) * The event parameter is ignored but needs to be non-NULL to work around a bug in old * kernel versions. */ - rc = epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, conn->sock, &event); + rc = epoll_ctl(g_poll_fd, EPOLL_CTL_DEL, conn->sock, &event); if (rc == 0) { return 0; } else { @@ -195,11 +303,11 @@ check_idle_conns(void) /* if nothing idle, can exit now */ if (STAILQ_EMPTY(&g_idle_conn_list_head)) { /* this epoll_wait is needed to finish socket closing process */ - epoll_wait(g_epoll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0); + epoll_wait(g_poll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0); } /* Perform a non-blocking epoll */ - nfds = epoll_wait(g_epoll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0); + nfds = epoll_wait(g_poll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0); if (nfds < 0) { SPDK_ERRLOG("epoll_wait failed! (ret: %d)\n", nfds); return; @@ -228,6 +336,8 @@ check_idle_conns(void) } } +#endif + int spdk_initialize_iscsi_conns(void) { size_t conns_size; @@ -242,7 +352,7 @@ int spdk_initialize_iscsi_conns(void) return -1; } - snprintf(g_shm_name, sizeof(g_shm_name), "spdk_iscsi_conns.%d", spdk_app_get_shm_id()); + snprintf(g_shm_name, sizeof(g_shm_name), "/spdk_iscsi_conns.%d", spdk_app_get_shm_id()); conns_array_fd = shm_open(g_shm_name, O_RDWR | O_CREAT, 0600); if (conns_array_fd < 0) { SPDK_ERRLOG("could not shm_open %s\n", g_shm_name);