net, iscsi: add struct spdk_sock abstraction

This provides an abstraction layer around TCP
sockets.  Previously we just used fd integers, but
we don't want to be tied to integers for alternative
userspace TCP stacks.

Future patches will do more work to enable multiple
implementations of this abstraction.  For now, just
get the abstraction in place for POSIX sockets and
make all of the iSCSI changes associated with it.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I9a825e9e02eb6927c8702d205665c626f57b3771

Reviewed-on: https://review.gerrithub.io/398861
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Ziye Yang <optimistyzy@gmail.com>

include/spdk/net.h

@@ -46,12 +46,14 @@
 extern "C" {
 #endif
 
+struct spdk_sock;
+
 int spdk_interface_init(void);
 void spdk_interface_destroy(void);
 
 const char *spdk_net_framework_get_name(void);
 int spdk_net_framework_start(void);
-void spdk_net_framework_clear_socket_association(int sock);
+void spdk_net_framework_clear_socket_association(struct spdk_sock *sock);
 void spdk_net_framework_fini(void);
 
 #define SPDK_IFNAMSIZE		32

include/spdk/sock.h

@@ -44,20 +44,22 @@
 extern "C" {
 #endif
 
-int spdk_sock_getaddr(int sock, char *saddr, int slen, char *caddr, int clen);
+struct spdk_sock;
-int spdk_sock_connect(const char *ip, int port);
-int spdk_sock_listen(const char *ip, int port);
-int spdk_sock_accept(int sock);
-int spdk_sock_close(int sock);
-ssize_t spdk_sock_recv(int sock, void *buf, size_t len);
-ssize_t spdk_sock_writev(int sock, struct iovec *iov, int iovcnt);
 
-int spdk_sock_set_recvlowat(int sock, int nbytes);
+int spdk_sock_getaddr(struct spdk_sock *sock, char *saddr, int slen, char *caddr, int clen);
-int spdk_sock_set_recvbuf(int sock, int sz);
+struct spdk_sock *spdk_sock_connect(const char *ip, int port);
-int spdk_sock_set_sendbuf(int sock, int sz);
+struct spdk_sock *spdk_sock_listen(const char *ip, int port);
+struct spdk_sock *spdk_sock_accept(struct spdk_sock *sock);
+int spdk_sock_close(struct spdk_sock **sock);
+ssize_t spdk_sock_recv(struct spdk_sock *sock, void *buf, size_t len);
+ssize_t spdk_sock_writev(struct spdk_sock *sock, struct iovec *iov, int iovcnt);
 
-bool spdk_sock_is_ipv6(int sock);
+int spdk_sock_set_recvlowat(struct spdk_sock *sock, int nbytes);
-bool spdk_sock_is_ipv4(int sock);
+int spdk_sock_set_recvbuf(struct spdk_sock *sock, int sz);
+int spdk_sock_set_sendbuf(struct spdk_sock *sock, int sz);
+
+bool spdk_sock_is_ipv6(struct spdk_sock *sock);
+bool spdk_sock_is_ipv4(struct spdk_sock *sock);
 
 #ifdef __cplusplus
 }

lib/iscsi/acceptor.c

@@ -49,19 +49,19 @@ static void
 spdk_iscsi_portal_accept(void *arg)
 {
 	struct spdk_iscsi_portal	*portal = arg;
-	int				rc, sock;
+	struct spdk_sock		*sock;
+	int				rc;
 
-	if (portal->sock < 0) {
+	if (portal->sock == NULL) {
 		return;
 	}
 
 	while (1) {
-		rc = spdk_sock_accept(portal->sock);
+		sock = spdk_sock_accept(portal->sock);
-		if (rc >= 0) {
+		if (sock != NULL) {
-			sock = rc;
 			rc = spdk_iscsi_conn_construct(portal, sock);
 			if (rc < 0) {
-				spdk_sock_close(sock);
+				spdk_sock_close(&sock);
 				SPDK_ERRLOG("spdk_iscsi_connection_construct() failed\n");
 				break;
 			}

lib/iscsi/conn.c

@@ -391,7 +391,7 @@ int spdk_initialize_iscsi_conns(void)
  */
 int
 spdk_iscsi_conn_construct(struct spdk_iscsi_portal *portal,
-			  int sock)
+			  struct spdk_sock *sock)
 {
 	struct spdk_iscsi_conn *conn;
 	int bufsize, i, rc;
@@ -692,7 +692,7 @@ void spdk_iscsi_conn_destruct(struct spdk_iscsi_conn *conn)
 	}
 
 	spdk_clear_all_transfer_task(conn, NULL);
-	spdk_sock_close(conn->sock);
+	spdk_sock_close(&conn->sock);
 	spdk_poller_unregister(&conn->logout_timer);
 	spdk_poller_unregister(&conn->flush_poller);
 
@@ -895,8 +895,8 @@ spdk_iscsi_conn_read_data(struct spdk_iscsi_conn *conn, int bytes,
 		if (errno == EAGAIN || errno == EWOULDBLOCK) {
 			return 0;
 		} else {
-			SPDK_ERRLOG("spdk_sock_recv() failed (fd=%d), errno %d: %s\n",
+			SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n",
-				    conn->sock, errno, spdk_strerror(errno));
+				    errno, spdk_strerror(errno));
 		}
 		return SPDK_ISCSI_CONNECTION_FATAL;
 	}
@@ -1152,8 +1152,8 @@ spdk_iscsi_conn_flush_pdus_internal(struct spdk_iscsi_conn *conn)
 		if (errno == EWOULDBLOCK || errno == EAGAIN) {
 			return 1;
 		} else {
-			SPDK_ERRLOG("spdk_sock_writev() failed (fd=%d), errno %d: %s\n",
+			SPDK_ERRLOG("spdk_sock_writev() failed, errno %d: %s\n",
-				    conn->sock, errno, spdk_strerror(errno));
+				    errno, spdk_strerror(errno));
 			return -1;
 		}
 	}

lib/iscsi/conn.h

@@ -83,7 +83,7 @@ struct spdk_iscsi_conn {
 	char				*portal_port;
 	struct spdk_cpuset		*portal_cpumask;
 	uint32_t			lcore;
-	int				sock;
+	struct spdk_sock		*sock;
 	struct spdk_iscsi_sess		*sess;
 
 	enum iscsi_connection_state	state;
@@ -171,7 +171,7 @@ extern struct spdk_iscsi_conn *g_conns_array;
 int spdk_initialize_iscsi_conns(void);
 void spdk_shutdown_iscsi_conns(void);
 
-int spdk_iscsi_conn_construct(struct spdk_iscsi_portal *portal, int sock);
+int spdk_iscsi_conn_construct(struct spdk_iscsi_portal *portal, struct spdk_sock *sock);
 void spdk_iscsi_conn_destruct(struct spdk_iscsi_conn *conn);
 void spdk_iscsi_conn_logout(struct spdk_iscsi_conn *conn);
 int spdk_iscsi_drop_conns(struct spdk_iscsi_conn *conn,

lib/iscsi/portal_grp.c

@@ -128,7 +128,7 @@ spdk_iscsi_portal_create(const char *host, const char *port, const char *cpumask
 
 	p->cpumask = core_mask;
 
-	p->sock = -1;
+	p->sock = NULL;
 	p->group = NULL; /* set at a later time by caller */
 	p->acceptor_poller = NULL;
 
@@ -175,9 +175,10 @@ spdk_iscsi_portal_destroy(struct spdk_iscsi_portal *p)
 static int
 spdk_iscsi_portal_open(struct spdk_iscsi_portal *p)
 {
-	int port, sock;
+	struct spdk_sock *sock;
+	int port;
 
-	if (p->sock >= 0) {
+	if (p->sock != NULL) {
 		SPDK_ERRLOG("portal (%s, %s) is already opened\n",
 			    p->host, p->port);
 		return -1;
@@ -185,7 +186,7 @@ spdk_iscsi_portal_open(struct spdk_iscsi_portal *p)
 
 	port = (int)strtol(p->port, NULL, 0);
 	sock = spdk_sock_listen(p->host, port);
-	if (sock < 0) {
+	if (sock == NULL) {
 		SPDK_ERRLOG("listen error %.64s.%d\n", p->host, port);
 		return -1;
 	}
@@ -207,12 +208,11 @@ spdk_iscsi_portal_open(struct spdk_iscsi_portal *p)
 static void
 spdk_iscsi_portal_close(struct spdk_iscsi_portal *p)
 {
-	if (p->sock >= 0) {
+	if (p->sock) {
 		SPDK_DEBUGLOG(SPDK_LOG_ISCSI, "close portal (%s, %s)\n",
 			      p->host, p->port);
 		spdk_iscsi_acceptor_stop(p);
-		spdk_sock_close(p->sock);
+		spdk_sock_close(&p->sock);
-		p->sock = -1;
 	}
 }
 

lib/iscsi/portal_grp.h

@@ -42,7 +42,7 @@ struct spdk_iscsi_portal {
 	struct spdk_iscsi_portal_grp	*group;
 	char				*host;
 	char				*port;
-	int				sock;
+	struct spdk_sock		*sock;
 	struct spdk_cpuset		*cpumask;
 	struct spdk_poller		*acceptor_poller;
 	TAILQ_ENTRY(spdk_iscsi_portal)	per_pg_tailq;

lib/net/net_framework_default.c

@@ -51,6 +51,6 @@ void spdk_net_framework_fini(void)
 }
 
 __attribute__((weak))
-void spdk_net_framework_clear_socket_association(int sock)
+void spdk_net_framework_clear_socket_association(struct spdk_sock *sock)
 {
 }

lib/net/sock.c

@@ -39,6 +39,10 @@
 #define MAX_TMPBUF 1024
 #define PORTNUMLEN 32
 
+struct spdk_sock {
+	int fd;
+};
+
 static int get_addr_str(struct sockaddr *sa, char *host, size_t hlen)
 {
 	const char *result = NULL;
@@ -68,15 +72,17 @@ static int get_addr_str(struct sockaddr *sa, char *host, size_t hlen)
 }
 
 int
-spdk_sock_getaddr(int sock, char *saddr, int slen, char *caddr, int clen)
+spdk_sock_getaddr(struct spdk_sock *sock, char *saddr, int slen, char *caddr, int clen)
 {
 	struct sockaddr_storage sa;
 	socklen_t salen;
 	int rc;
 
+	assert(sock != NULL);
+
 	memset(&sa, 0, sizeof sa);
 	salen = sizeof sa;
-	rc = getsockname(sock, (struct sockaddr *) &sa, &salen);
+	rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen);
 	if (rc != 0) {
 		SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno);
 		return -1;
@@ -103,7 +109,7 @@ spdk_sock_getaddr(int sock, char *saddr, int slen, char *caddr, int clen)
 
 	memset(&sa, 0, sizeof sa);
 	salen = sizeof sa;
-	rc = getpeername(sock, (struct sockaddr *) &sa, &salen);
+	rc = getpeername(sock->fd, (struct sockaddr *) &sa, &salen);
 	if (rc != 0) {
 		SPDK_ERRLOG("getpeername() failed (errno=%d)\n", errno);
 		return -1;
@@ -123,9 +129,10 @@ enum spdk_sock_create_type {
 	SPDK_SOCK_CREATE_CONNECT,
 };
 
-static int
+static struct spdk_sock *
 spdk_sock_create(const char *ip, int port, enum spdk_sock_create_type type)
 {
+	struct spdk_sock *sock;
 	char buf[MAX_TMPBUF];
 	char portnum[PORTNUMLEN];
 	char *p;
@@ -135,7 +142,7 @@ spdk_sock_create(const char *ip, int port, enum spdk_sock_create_type type)
 	int rc;
 
 	if (ip == NULL) {
-		return -1;
+		return NULL;
 	}
 	if (ip[0] == '[') {
 		snprintf(buf, sizeof(buf), "%s", ip + 1);
@@ -156,7 +163,7 @@ spdk_sock_create(const char *ip, int port, enum spdk_sock_create_type type)
 	rc = getaddrinfo(ip, portnum, &hints, &res0);
 	if (rc != 0) {
 		SPDK_ERRLOG("getaddrinfo() failed (errno=%d)\n", errno);
-		return -1;
+		return NULL;
 	}
 
 	/* try listen */
@@ -234,60 +241,114 @@ retry:
 	freeaddrinfo(res0);
 
 	if (fd < 0) {
-		return -1;
+		return NULL;
 	}
-	return fd;
+
+	sock = calloc(1, sizeof(*sock));
+	if (sock == NULL) {
+		SPDK_ERRLOG("sock allocation failed\n");
+		close(fd);
+		return NULL;
+	}
+
+	sock->fd = fd;
+	return sock;
 }
 
-int
+struct spdk_sock *
 spdk_sock_listen(const char *ip, int port)
 {
 	return spdk_sock_create(ip, port, SPDK_SOCK_CREATE_LISTEN);
 }
 
-int
+struct spdk_sock *
 spdk_sock_connect(const char *ip, int port)
 {
 	return spdk_sock_create(ip, port, SPDK_SOCK_CREATE_CONNECT);
 }
 
-int
+struct spdk_sock *
-spdk_sock_accept(int sock)
+spdk_sock_accept(struct spdk_sock *sock)
 {
 	struct sockaddr_storage		sa;
 	socklen_t			salen;
+	int				rc;
+	struct spdk_sock		*new_sock;
 
 	memset(&sa, 0, sizeof(sa));
 	salen = sizeof(sa);
-	return accept(sock, (struct sockaddr *)&sa, &salen);
+
+	assert(sock != NULL);
+
+	rc = accept(sock->fd, (struct sockaddr *)&sa, &salen);
+
+	if (rc == -1) {
+		return NULL;
+	}
+
+	new_sock = calloc(1, sizeof(*sock));
+	if (new_sock == NULL) {
+		SPDK_ERRLOG("sock allocation failed\n");
+		close(rc);
+		return NULL;
+	}
+
+	new_sock->fd = rc;
+	return new_sock;
 }
 
 int
-spdk_sock_close(int sock)
+spdk_sock_close(struct spdk_sock **sock)
 {
-	return close(sock);
+	int rc;
+
+	if (*sock == NULL) {
+		errno = EBADF;
+		return -1;
+	}
+
+	rc = close((*sock)->fd);
+
+	if (rc == 0) {
+		free(*sock);
+		*sock = NULL;
+	}
+
+	return rc;
 }
 
 ssize_t
-spdk_sock_recv(int sock, void *buf, size_t len)
+spdk_sock_recv(struct spdk_sock *sock, void *buf, size_t len)
 {
-	return recv(sock, buf, len, MSG_DONTWAIT);
+	if (sock == NULL) {
+		errno = EBADF;
+		return -1;
+	}
+
+	return recv(sock->fd, buf, len, MSG_DONTWAIT);
 }
 
 ssize_t
-spdk_sock_writev(int sock, struct iovec *iov, int iovcnt)
+spdk_sock_writev(struct spdk_sock *sock, struct iovec *iov, int iovcnt)
 {
-	return writev(sock, iov, iovcnt);
+	if (sock == NULL) {
+		errno = EBADF;
+		return -1;
+	}
+
+	return writev(sock->fd, iov, iovcnt);
 }
 
 int
-spdk_sock_set_recvlowat(int s, int nbytes)
+spdk_sock_set_recvlowat(struct spdk_sock *sock, int nbytes)
 {
 	int val;
 	int rc;
 
+	assert(sock != NULL);
+
 	val = nbytes;
-	rc = setsockopt(s, SOL_SOCKET, SO_RCVLOWAT, &val, sizeof val);
+	rc = setsockopt(sock->fd, SOL_SOCKET, SO_RCVLOWAT, &val, sizeof val);
 	if (rc != 0) {
 		return -1;
 	}
@@ -295,29 +356,35 @@ spdk_sock_set_recvlowat(int s, int nbytes)
 }
 
 int
-spdk_sock_set_recvbuf(int sock, int sz)
+spdk_sock_set_recvbuf(struct spdk_sock *sock, int sz)
 {
-	return setsockopt(sock, SOL_SOCKET, SO_RCVBUF,
+	assert(sock != NULL);
+
+	return setsockopt(sock->fd, SOL_SOCKET, SO_RCVBUF,
 			  &sz, sizeof(sz));
 }
 
 int
-spdk_sock_set_sendbuf(int sock, int sz)
+spdk_sock_set_sendbuf(struct spdk_sock *sock, int sz)
 {
-	return setsockopt(sock, SOL_SOCKET, SO_SNDBUF,
+	assert(sock != NULL);
+
+	return setsockopt(sock->fd, SOL_SOCKET, SO_SNDBUF,
 			  &sz, sizeof(sz));
 }
 
 bool
-spdk_sock_is_ipv6(int sock)
+spdk_sock_is_ipv6(struct spdk_sock *sock)
 {
 	struct sockaddr_storage sa;
 	socklen_t salen;
 	int rc;
 
+	assert(sock != NULL);
+
 	memset(&sa, 0, sizeof sa);
 	salen = sizeof sa;
-	rc = getsockname(sock, (struct sockaddr *) &sa, &salen);
+	rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen);
 	if (rc != 0) {
 		SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno);
 		return false;
@@ -327,15 +394,17 @@ spdk_sock_is_ipv6(int sock)
 }
 
 bool
-spdk_sock_is_ipv4(int sock)
+spdk_sock_is_ipv4(struct spdk_sock *sock)
 {
 	struct sockaddr_storage sa;
 	socklen_t salen;
 	int rc;
 
+	assert(sock != NULL);
+
 	memset(&sa, 0, sizeof sa);
 	salen = sizeof sa;
-	rc = getsockname(sock, (struct sockaddr *) &sa, &salen);
+	rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen);
 	if (rc != 0) {
 		SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno);
 		return false;

test/unit/lib/iscsi/common.c

@@ -114,15 +114,19 @@ spdk_iscsi_acceptor_stop(struct spdk_iscsi_portal *p)
 {
 }
 
-int
+struct spdk_sock *
 spdk_sock_listen(const char *ip, int port)
 {
-	return 0;
+	static int g_sock;
+
+	return (struct spdk_sock *)&g_sock;
 }
 
 int
-spdk_sock_close(int sock)
+spdk_sock_close(struct spdk_sock **sock)
 {
+	*sock = NULL;
+
 	return 0;
 }
 

test/unit/lib/iscsi/conn.c/conn_ut.c

@@ -98,43 +98,44 @@ spdk_event_call(struct spdk_event *event)
 }
 
 int
-spdk_sock_getaddr(int sock, char *saddr, int slen, char *caddr, int clen)
+spdk_sock_getaddr(struct spdk_sock *sock, char *saddr, int slen, char *caddr, int clen)
 {
 	return 0;
 }
 
 int
-spdk_sock_close(int sock)
+spdk_sock_close(struct spdk_sock **sock)
+{
+	*sock = NULL;
+	return 0;
+}
+
+ssize_t
+spdk_sock_recv(struct spdk_sock *sock, void *buf, size_t len)
 {
 	return 0;
 }
 
 ssize_t
-spdk_sock_recv(int sock, void *buf, size_t len)
+spdk_sock_writev(struct spdk_sock *sock, struct iovec *iov, int iovcnt)
-{
-	return 0;
-}
-
-ssize_t
-spdk_sock_writev(int sock, struct iovec *iov, int iovcnt)
 {
 	return 0;
 }
 
 int
-spdk_sock_set_recvlowat(int s, int nbytes)
+spdk_sock_set_recvlowat(struct spdk_sock *s, int nbytes)
 {
 	return 0;
 }
 
 int
-spdk_sock_set_recvbuf(int sock, int sz)
+spdk_sock_set_recvbuf(struct spdk_sock *sock, int sz)
 {
 	return 0;
 }
 
 int
-spdk_sock_set_sendbuf(int sock, int sz)
+spdk_sock_set_sendbuf(struct spdk_sock *sock, int sz)
 {
 	return 0;
 }

test/unit/lib/iscsi/tgt_node.c/tgt_node_ut.c

@@ -62,13 +62,13 @@ DEFINE_STUB(spdk_scsi_lun_get_id,
 	    0);
 
 bool
-spdk_sock_is_ipv6(int sock)
+spdk_sock_is_ipv6(struct spdk_sock *sock)
 {
 	return false;
 }
 
 bool
-spdk_sock_is_ipv4(int sock)
+spdk_sock_is_ipv4(struct spdk_sock *sock)
 {
 	return false;
 }