Make iscsi app to work on freebsd

Change-Id: Ic4221e044eb5af92aa06e468eb989bee285022af
Signed-off-by: Jan Kryl <jan.kryl@nexenta.com>
Reviewed-on: https://review.gerrithub.io/371303
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

app/Makefile

@@ -37,8 +37,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 DIRS-y += trace
 DIRS-y += nvmf_tgt
 DIRS-y += iscsi_top
-ifeq ($(OS),Linux)
 DIRS-y += iscsi_tgt
+ifeq ($(OS),Linux)
 DIRS-$(CONFIG_VHOST) += vhost
 endif
 

app/iscsi_top/iscsi_top.cpp

@@ -79,7 +79,7 @@ print_connections(void)
 	int			fd, i;
 	char			shm_name[64];
 
-	snprintf(shm_name, sizeof(shm_name), "spdk_iscsi_conns.%d", g_shm_id);
+	snprintf(shm_name, sizeof(shm_name), "/spdk_iscsi_conns.%d", g_shm_id);
 	fd = shm_open(shm_name, O_RDONLY, 0600);
 	if (fd < 0) {
 		fprintf(stderr, "Cannot open shared memory: %s\n", shm_name);

lib/Makefile

@@ -36,9 +36,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 DIRS-y += bdev blob blobfs conf copy cunit event json jsonrpc \
           log env_dpdk net rpc trace util nvme nvmf scsi ioat \
-	  ut_mock
+	  ut_mock iscsi
 ifeq ($(OS),Linux)
-DIRS-y += iscsi
 DIRS-$(CONFIG_VHOST) += vhost
 endif
 

lib/iscsi/conn.c

@@ -37,7 +37,11 @@
 #include <rte_config.h>
 #include <rte_mempool.h>
 
+#if defined(__FreeBSD__)
+#include <sys/event.h>
+#else
 #include <sys/epoll.h>
+#endif
 
 #include "spdk/endian.h"
 #include "spdk/env.h"
@@ -77,7 +81,7 @@ static uint32_t spdk_iscsi_conn_allocate_reactor(uint64_t cpumask);
 static void __add_idle_conn(void *arg1, void *arg2);
 
 /** Global variables used for managing idle connections. */
-static int g_epoll_fd = 0;
+static int g_poll_fd = 0;
 static struct spdk_poller *g_idle_conn_poller;
 static STAILQ_HEAD(idle_list, spdk_iscsi_conn) g_idle_conn_list_head;
 
@@ -133,12 +137,116 @@ spdk_find_iscsi_connection_by_id(int cid)
 	}
 }
 
+#if defined(__FreeBSD__)
+
 static int
 init_idle_conns(void)
 {
-	assert(g_epoll_fd == 0);
+	assert(g_poll_fd == 0);
-	g_epoll_fd = epoll_create1(0);
+	g_poll_fd = kqueue();
-	if (g_epoll_fd < 0) {
+	if (g_poll_fd < 0) {
+		SPDK_ERRLOG("kqueue failed master lcore\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+add_idle_conn(struct spdk_iscsi_conn *conn)
+{
+	struct kevent event;
+	struct timespec ts = {0};
+	int rc;
+
+	EV_SET(&event, conn->sock, EVFILT_READ, EV_ADD, 0, 0, conn);
+
+	rc = kevent(g_poll_fd, &event, 1, NULL, 0, &ts);
+	if (rc == -1) {
+		SPDK_ERRLOG("kevent(EV_ADD) failed\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+del_idle_conn(struct spdk_iscsi_conn *conn)
+{
+	struct kevent event;
+	struct timespec ts = {0};
+	char buf[64];
+	int rc;
+
+	EV_SET(&event, conn->sock, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+
+	rc = kevent(g_poll_fd, &event, 1, NULL, 0, &ts);
+	if (rc == -1) {
+		SPDK_ERRLOG("kevent(EV_DELETE) failed\n");
+		return -1;
+	}
+	if (event.flags & EV_ERROR) {
+		strerror_r(event.data, buf, sizeof(buf));
+		SPDK_ERRLOG("kevent(EV_DELETE) failed: %s\n", buf);
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+check_idle_conns(void)
+{
+	struct kevent events[SPDK_MAX_POLLERS_PER_CORE];
+	int i;
+	int nfds;
+	struct spdk_iscsi_conn *conn;
+	struct timespec ts = {0};
+
+	/* if nothing idle, can exit now */
+	if (STAILQ_EMPTY(&g_idle_conn_list_head)) {
+		/* this kevent is needed to finish socket closing process */
+		kevent(g_poll_fd, NULL, 0, events, SPDK_MAX_POLLERS_PER_CORE, &ts);
+	}
+
+	/* Perform a non-blocking poll */
+	nfds = kevent(g_poll_fd, NULL, 0, events, SPDK_MAX_POLLERS_PER_CORE, &ts);
+	if (nfds < 0) {
+		SPDK_ERRLOG("kevent failed! (ret: %d)\n", nfds);
+		return;
+	}
+
+	if (nfds > SPDK_MAX_POLLERS_PER_CORE) {
+		SPDK_ERRLOG("kevent events exceeded limit! %d > %d\n", nfds,
+			    SPDK_MAX_POLLERS_PER_CORE);
+		assert(0);
+	}
+
+	/*
+	 * In the case of any event cause (EPOLLIN or EPOLLERR)
+	 * just make the connection active for normal process loop.
+	 */
+	for (i = 0; i < nfds; i++) {
+
+		conn = (struct spdk_iscsi_conn *)events[i].udata;
+
+		/*
+		 * Flag the connection that an event was noticed
+		 * such that during the list scan process it will
+		 * be re-inserted into the active ring
+		 */
+		conn->pending_activate_event = true;
+	}
+}
+
+#else
+
+static int
+init_idle_conns(void)
+{
+	assert(g_poll_fd == 0);
+	g_poll_fd = epoll_create1(0);
+	if (g_poll_fd < 0) {
 		SPDK_ERRLOG("epoll_create1 failed master lcore\n");
 		return -1;
 	}
@@ -156,7 +264,7 @@ add_idle_conn(struct spdk_iscsi_conn *conn)
 	event.data.u64 = 0LL;
 	event.data.ptr = conn;
 
-	rc = epoll_ctl(g_epoll_fd, EPOLL_CTL_ADD, conn->sock, &event);
+	rc = epoll_ctl(g_poll_fd, EPOLL_CTL_ADD, conn->sock, &event);
 	if (rc == 0) {
 		return 0;
 	} else {
@@ -175,7 +283,7 @@ del_idle_conn(struct spdk_iscsi_conn *conn)
 	 * The event parameter is ignored but needs to be non-NULL to work around a bug in old
 	 * kernel versions.
 	 */
-	rc = epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, conn->sock, &event);
+	rc = epoll_ctl(g_poll_fd, EPOLL_CTL_DEL, conn->sock, &event);
 	if (rc == 0) {
 		return 0;
 	} else {
@@ -195,11 +303,11 @@ check_idle_conns(void)
 	/* if nothing idle, can exit now */
 	if (STAILQ_EMPTY(&g_idle_conn_list_head)) {
 		/* this epoll_wait is needed to finish socket closing process */
-		epoll_wait(g_epoll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0);
+		epoll_wait(g_poll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0);
 	}
 
 	/* Perform a non-blocking epoll */
-	nfds = epoll_wait(g_epoll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0);
+	nfds = epoll_wait(g_poll_fd, events, SPDK_MAX_POLLERS_PER_CORE, 0);
 	if (nfds < 0) {
 		SPDK_ERRLOG("epoll_wait failed! (ret: %d)\n", nfds);
 		return;
@@ -228,6 +336,8 @@ check_idle_conns(void)
 	}
 }
 
+#endif
+
 int spdk_initialize_iscsi_conns(void)
 {
 	size_t conns_size;
@@ -242,7 +352,7 @@ int spdk_initialize_iscsi_conns(void)
 		return -1;
 	}
 
-	snprintf(g_shm_name, sizeof(g_shm_name), "spdk_iscsi_conns.%d", spdk_app_get_shm_id());
+	snprintf(g_shm_name, sizeof(g_shm_name), "/spdk_iscsi_conns.%d", spdk_app_get_shm_id());
 	conns_array_fd = shm_open(g_shm_name, O_RDWR | O_CREAT, 0600);
 	if (conns_array_fd < 0) {
 		SPDK_ERRLOG("could not shm_open %s\n", g_shm_name);