nvmf: Add a public API header

This changes as little code as possible while still creating
a single public API header. This enables future clean up
of the public API and clarification of the exposed
concepts.

Change-Id: I780e7a5a9afd27acf0276516bd71b896ad301c50
Signed-off-by: Ben Walker <benjamin.walker@intel.com>

app/nvmf_tgt/Makefile

@@ -39,10 +39,6 @@ APP = nvmf_tgt
 
 CFLAGS += $(ENV_CFLAGS)
 
-# Add NVMf library directory to include path
-# TODO: remove this once NVMf has a public API header
-CFLAGS += -I$(SPDK_ROOT_DIR)/lib
-
 C_SRCS := conf.c nvmf_tgt.c nvmf_rpc.c
 
 SPDK_LIBS = \

app/nvmf_tgt/conf.c

@@ -44,16 +44,17 @@
 #include <rte_lcore.h>
 
 #include "nvmf_tgt.h"
-#include "nvmf/subsystem.h"
+
-#include "nvmf/transport.h"
 #include "spdk/conf.h"
 #include "spdk/log.h"
 #include "spdk/bdev.h"
+#include "spdk/nvme.h"
+#include "spdk/nvmf.h"
 
 #define MAX_LISTEN_ADDRESSES 255
 #define MAX_HOSTS 255
-
 #define PORTNUMSTRLEN 32
+#define SPDK_NVMF_DEFAULT_SIN_PORT ((uint16_t)4420)
 
 #define ACCEPT_TIMEOUT_US		1000 /* 1ms */
 
@@ -142,6 +143,7 @@ spdk_add_nvmf_discovery_subsystem(void)
 	struct nvmf_tgt_subsystem *app_subsys;
 
 	app_subsys = nvmf_tgt_create_subsystem(0, SPDK_NVMF_DISCOVERY_NQN, SPDK_NVMF_SUBTYPE_DISCOVERY,
+					       NVMF_SUBSYSTEM_MODE_DIRECT,
 					       rte_get_master_lcore());
 	if (app_subsys == NULL) {
 		SPDK_ERRLOG("Failed creating discovery nvmf library subsystem\n");
@@ -372,8 +374,8 @@ attach_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr *ctr
 	char path[MAX_STRING_LEN];
 	int numa_node = -1;
 
-	SPDK_NOTICELOG("Attaching NVMe device %x:%x:%x.%x to subsystem %s\n",
+	SPDK_NOTICELOG("Attaching NVMe device %p at %x:%x:%x.%x to subsystem %p\n",
-		       found_domain, found_bus, found_dev, found_func, ctx->subsystem->subnqn);
+		       ctrlr, found_domain, found_bus, found_dev, found_func, ctx->subsystem);
 
 	snprintf(path, sizeof(path), "/sys/bus/pci/devices/%04x:%02x:%02x.%1u/numa_node",
 		 found_domain, found_bus, found_dev, found_func);
@@ -433,9 +435,10 @@ spdk_nvmf_allocate_lcore(uint64_t mask, uint32_t lcore)
 static int
 spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 {
-	const char *nqn, *mode;
+	const char *nqn, *mode_str;
 	struct nvmf_tgt_subsystem *app_subsys;
 	struct spdk_nvmf_subsystem *subsystem;
+	enum spdk_nvmf_subsystem_mode mode;
 	int i, ret;
 	uint64_t mask;
 	int lcore = 0;
@@ -458,28 +461,28 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 	}
 	lcore = spdk_nvmf_allocate_lcore(mask, lcore);
 
-	app_subsys = nvmf_tgt_create_subsystem(sp->num, nqn, SPDK_NVMF_SUBTYPE_NVME, lcore);
+	mode_str = spdk_conf_section_get_val(sp, "Mode");
+	if (mode_str == NULL) {
+		SPDK_ERRLOG("No Mode specified for Subsystem %d\n", sp->num);
+		return -1;
+	}
+
+	if (strcasecmp(mode_str, "Direct") == 0) {
+		mode = NVMF_SUBSYSTEM_MODE_DIRECT;
+	} else if (strcasecmp(mode_str, "Virtual") == 0) {
+		mode = NVMF_SUBSYSTEM_MODE_VIRTUAL;
+	} else {
+		SPDK_ERRLOG("Invalid Subsystem mode: %s\n", mode_str);
+		return -1;
+	}
+
+	app_subsys = nvmf_tgt_create_subsystem(sp->num, nqn, SPDK_NVMF_SUBTYPE_NVME, mode, lcore);
 	if (app_subsys == NULL) {
 		SPDK_ERRLOG("Subsystem createion failed\n");
 		return -1;
 	}
 	subsystem = app_subsys->subsystem;
 
-	mode = spdk_conf_section_get_val(sp, "Mode");
-	if (mode == NULL) {
-		SPDK_ERRLOG("No Mode specified for Subsystem %d\n", sp->num);
-		return -1;
-	}
-
-	if (strcasecmp(mode, "Direct") == 0) {
-		subsystem->mode = NVMF_SUBSYSTEM_MODE_DIRECT;
-	} else if (strcasecmp(mode, "Virtual") == 0) {
-		subsystem->mode = NVMF_SUBSYSTEM_MODE_VIRTUAL;
-	} else {
-		SPDK_ERRLOG("Invalid Subsystem mode: %s\n", mode);
-		return -1;
-	}
-
 	/* Parse Listen sections */
 	for (i = 0; i < MAX_LISTEN_ADDRESSES; i++) {
 		char *transport_name, *listen_addr;
@@ -533,7 +536,7 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 		spdk_nvmf_subsystem_add_host(subsystem, host_nqn);
 	}
 
-	if (subsystem->mode == NVMF_SUBSYSTEM_MODE_DIRECT) {
+	if (mode == NVMF_SUBSYSTEM_MODE_DIRECT) {
 		const char *bdf;
 		struct spdk_nvmf_probe_ctx ctx = { 0 };
 
@@ -584,9 +587,7 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 			return -1;
 		}
 
-		subsystem->dev.virt.ns_count = 0;
+		spdk_nvmf_subsystem_set_sn(subsystem, sn);
-		snprintf(subsystem->dev.virt.sn, MAX_SN_LEN, "%s", sn);
-		subsystem->ops = &spdk_nvmf_virtual_ctrlr_ops;
 
 		for (i = 0; i < MAX_VIRTUAL_NAMESPACE; i++) {
 			val = spdk_conf_section_get_nval(sp, "Namespace", i);
@@ -609,8 +610,8 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 				return -1;
 			}
 
-			SPDK_NOTICELOG("Attaching block device %s to subsystem %s\n",
+			SPDK_NOTICELOG("Attaching block device %s to subsystem %p\n",
-				       bdev->name, subsystem->subnqn);
+				       bdev->name, subsystem);
 
 		}
 	}
@@ -661,13 +662,14 @@ spdk_nvmf_parse_conf(void)
 
 int
 spdk_nvmf_parse_subsystem_for_rpc(const char *name,
-				  const char *mode, uint32_t lcore,
+				  const char *mode_str, uint32_t lcore,
 				  int num_listen_addresses, struct rpc_listen_address *addresses,
 				  int num_hosts, char *hosts[], const char *bdf,
 				  const char *sn, int num_devs, char *dev_list[])
 {
 	struct spdk_nvmf_subsystem *subsystem;
 	struct nvmf_tgt_subsystem *app_subsys;
+	enum spdk_nvmf_subsystem_mode mode;
 	int i, ret;
 	uint64_t mask;
 	int num = 0;
@@ -699,27 +701,29 @@ spdk_nvmf_parse_subsystem_for_rpc(const char *name,
 	mask = spdk_app_get_core_mask();
 	lcore = spdk_nvmf_allocate_lcore(mask, lcore);
 
-	app_subsys = nvmf_tgt_create_subsystem(num, name, SPDK_NVMF_SUBTYPE_NVME, lcore);
+	/* Determine the mode the subsysem will operate in */
+	if (mode_str == NULL) {
+		SPDK_ERRLOG("No Mode specified for Subsystem %d\n", num);
+		return -1;
+	}
+
+	if (strcasecmp(mode_str, "Direct") == 0) {
+		mode = NVMF_SUBSYSTEM_MODE_DIRECT;
+	} else if (strcasecmp(mode_str, "Virtual") == 0) {
+		mode = NVMF_SUBSYSTEM_MODE_VIRTUAL;
+	} else {
+		SPDK_ERRLOG("Invalid Subsystem mode: %s\n", mode_str);
+		return -1;
+	}
+
+	app_subsys = nvmf_tgt_create_subsystem(num, name, SPDK_NVMF_SUBTYPE_NVME,
+					       mode, lcore);
 	if (app_subsys == NULL) {
 		SPDK_ERRLOG("Subsystem creation failed\n");
 		return -1;
 	}
 	subsystem = app_subsys->subsystem;
 
-	if (mode == NULL) {
-		SPDK_ERRLOG("No Mode specified for Subsystem %d\n", num);
-		return -1;
-	}
-
-	if (strcasecmp(mode, "Direct") == 0) {
-		subsystem->mode = NVMF_SUBSYSTEM_MODE_DIRECT;
-	} else if (strcasecmp(mode, "Virtual") == 0) {
-		subsystem->mode = NVMF_SUBSYSTEM_MODE_VIRTUAL;
-	} else {
-		SPDK_ERRLOG("Invalid Subsystem mode: %s\n", mode);
-		return -1;
-	}
-
 	/* Parse Listen sections */
 	for (i = 0; i < num_listen_addresses; i++) {
 		const struct spdk_nvmf_transport *transport;
@@ -738,7 +742,7 @@ spdk_nvmf_parse_subsystem_for_rpc(const char *name,
 		spdk_nvmf_subsystem_add_host(subsystem, hosts[i]);
 	}
 
-	if (subsystem->mode == NVMF_SUBSYSTEM_MODE_DIRECT) {
+	if (mode == NVMF_SUBSYSTEM_MODE_DIRECT) {
 		struct spdk_nvmf_probe_ctx ctx = { 0 };
 
 		if (bdf == NULL) {
@@ -791,7 +795,6 @@ spdk_nvmf_parse_subsystem_for_rpc(const char *name,
 
 		subsystem->dev.virt.ns_count = 0;
 		snprintf(subsystem->dev.virt.sn, MAX_SN_LEN, "%s", sn);
-		subsystem->ops = &spdk_nvmf_virtual_ctrlr_ops;
 
 		for (i = 0; i < num_devs; i++) {
 			namespace = dev_list[i];

app/nvmf_tgt/nvmf_rpc.c

@@ -34,24 +34,26 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-#include "nvmf_tgt.h"
+#include "spdk/bdev.h"
-#include "nvmf/subsystem.h"
-#include "nvmf/transport.h"
 #include "spdk/log.h"
 #include "spdk/rpc.h"
 #include "spdk/env.h"
 #include "spdk/nvme.h"
+#include "spdk/nvmf.h"
+
+#include "nvmf_tgt.h"
 
 static void
-dump_nvmf_subsystem(struct spdk_json_write_ctx *w, struct spdk_nvmf_subsystem *subsystem)
+dump_nvmf_subsystem(struct spdk_json_write_ctx *w, struct nvmf_tgt_subsystem *tgt_subsystem)
 {
-	struct spdk_nvmf_listen_addr *listen_addr;
+	struct spdk_nvmf_listen_addr 	*listen_addr;
 	struct spdk_nvmf_host		*host;
+	struct spdk_nvmf_subsystem	*subsystem = tgt_subsystem->subsystem;
 
 	spdk_json_write_object_begin(w);
 
 	spdk_json_write_name(w, "core");
-	spdk_json_write_int32(w, subsystem->lcore);
+	spdk_json_write_int32(w, tgt_subsystem->lcore);
 
 	spdk_json_write_name(w, "nqn");
 	spdk_json_write_string(w, subsystem->subnqn);
@@ -152,7 +154,7 @@ spdk_rpc_get_nvmf_subsystems(struct spdk_jsonrpc_server_conn *conn,
 	spdk_json_write_array_begin(w);
 	tgt_subsystem = nvmf_tgt_subsystem_first();
 	while (tgt_subsystem) {
-		dump_nvmf_subsystem(w, tgt_subsystem->subsystem);
+		dump_nvmf_subsystem(w, tgt_subsystem);
 		tgt_subsystem = nvmf_tgt_subsystem_next(tgt_subsystem);
 	}
 	spdk_json_write_array_end(w);

app/nvmf_tgt/nvmf_tgt.c

@@ -43,13 +43,8 @@
 
 #include "nvmf_tgt.h"
 
+#include "spdk/bdev.h"
 #include "spdk/event.h"
-
-#include "nvmf/transport.h"
-#include "nvmf/subsystem.h"
-#include "nvmf/request.h"
-#include "nvmf/session.h"
-
 #include "spdk/log.h"
 #include "spdk/nvme.h"
 #include "spdk/io_channel.h"
@@ -221,7 +216,8 @@ nvmf_tgt_start_subsystem(struct nvmf_tgt_subsystem *app_subsys)
 }
 
 struct nvmf_tgt_subsystem *
-nvmf_tgt_create_subsystem(int num, const char *name, enum spdk_nvmf_subtype subtype, uint32_t lcore)
+nvmf_tgt_create_subsystem(int num, const char *name, enum spdk_nvmf_subtype subtype,
+			  enum spdk_nvmf_subsystem_mode mode, uint32_t lcore)
 {
 	struct spdk_nvmf_subsystem *subsystem;
 	struct nvmf_tgt_subsystem *app_subsys;
@@ -232,7 +228,8 @@ nvmf_tgt_create_subsystem(int num, const char *name, enum spdk_nvmf_subtype subt
 		return NULL;
 	}
 
-	subsystem = spdk_nvmf_create_subsystem(num, name, subtype, app_subsys, connect_cb, disconnect_cb);
+	subsystem = spdk_nvmf_create_subsystem(num, name, subtype, mode, app_subsys, connect_cb,
+					       disconnect_cb);
 	if (subsystem == NULL) {
 		SPDK_ERRLOG("Subsystem creation failed\n");
 		free(app_subsys);

app/nvmf_tgt/nvmf_tgt.h

@@ -36,7 +36,7 @@
 
 #include <stdint.h>
 
-#include "spdk/nvmf_spec.h"
+#include "spdk/nvmf.h"
 #include "spdk/queue.h"
 
 struct rpc_listen_address {
@@ -74,6 +74,7 @@ void nvmf_tgt_start_subsystem(struct nvmf_tgt_subsystem *subsystem);
 struct nvmf_tgt_subsystem *nvmf_tgt_create_subsystem(int num,
 		const char *name,
 		enum spdk_nvmf_subtype subtype,
+		enum spdk_nvmf_subsystem_mode mode,
 		uint32_t lcore);
 
 int

include/spdk/nvmf.h

@@ -0,0 +1,274 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * NVMe over Fabrics target public API
+ */
+
+#ifndef SPDK_NVMF_H
+#define SPDK_NVMF_H
+
+#include <stdint.h>
+
+#include "spdk/nvmf_spec.h"
+#include "spdk/queue.h"
+
+#define MAX_VIRTUAL_NAMESPACE 16
+#define MAX_SN_LEN 20
+
+#define nvmf_min(a,b) (((a)<(b))?(a):(b))
+#define nvmf_max(a,b) (((a)>(b))?(a):(b))
+
+int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess,
+		  uint32_t in_capsule_data_size, uint32_t max_io_size);
+
+int nvmf_tgt_fini(void);
+
+int spdk_nvmf_check_pools(void);
+
+struct spdk_nvmf_subsystem;
+struct spdk_nvmf_session;
+struct spdk_nvmf_conn;
+struct spdk_nvmf_request;
+struct spdk_bdev;
+struct spdk_nvme_ctrlr;
+struct spdk_nvmf_transport;
+struct spdk_nvmf_request;
+struct spdk_nvmf_conn;
+struct spdk_pci_device;
+
+typedef void (*spdk_nvmf_subsystem_connect_fn)(void *cb_ctx, struct spdk_nvmf_request *req);
+typedef void (*spdk_nvmf_subsystem_disconnect_fn)(void *cb_ctx, struct spdk_nvmf_conn *conn);
+
+enum spdk_nvmf_subsystem_mode {
+	NVMF_SUBSYSTEM_MODE_DIRECT	= 0,
+	NVMF_SUBSYSTEM_MODE_VIRTUAL	= 1,
+};
+
+struct spdk_nvmf_listen_addr {
+	char					*traddr;
+	char					*trsvcid;
+	const struct spdk_nvmf_transport	*transport;
+	TAILQ_ENTRY(spdk_nvmf_listen_addr)	link;
+};
+
+struct spdk_nvmf_host {
+	char				*nqn;
+	TAILQ_ENTRY(spdk_nvmf_host)	link;
+};
+
+struct spdk_nvmf_ctrlr_ops {
+	/**
+	 * Get NVMe identify controller data.
+	 */
+	void (*ctrlr_get_data)(struct spdk_nvmf_session *session);
+
+	/**
+	 * Process admin command.
+	 */
+	int (*process_admin_cmd)(struct spdk_nvmf_request *req);
+
+	/**
+	 * Process IO command.
+	 */
+	int (*process_io_cmd)(struct spdk_nvmf_request *req);
+
+	/**
+	 * Poll for completions.
+	 */
+	void (*poll_for_completions)(struct spdk_nvmf_session *session);
+
+	/**
+	 * Detach the controller.
+	 */
+	void (*detach)(struct spdk_nvmf_subsystem *subsystem);
+};
+
+/*
+ * The NVMf subsystem, as indicated in the specification, is a collection
+ * of virtual controller sessions.  Any individual controller session has
+ * access to all the NVMe device/namespaces maintained by the subsystem.
+ */
+struct spdk_nvmf_subsystem {
+	uint16_t num;
+	uint32_t lcore;
+	char subnqn[SPDK_NVMF_NQN_MAX_LEN];
+	enum spdk_nvmf_subsystem_mode mode;
+	enum spdk_nvmf_subtype subtype;
+
+	union {
+		struct {
+			struct spdk_nvme_ctrlr *ctrlr;
+			struct spdk_nvme_qpair *io_qpair;
+			struct spdk_pci_device *pci_dev;
+		} direct;
+
+		struct {
+			char	sn[MAX_SN_LEN + 1];
+			struct spdk_bdev *ns_list[MAX_VIRTUAL_NAMESPACE];
+			struct spdk_io_channel *ch[MAX_VIRTUAL_NAMESPACE];
+			uint16_t ns_count;
+		} virt;
+	} dev;
+
+	const struct spdk_nvmf_ctrlr_ops *ops;
+
+	void					*cb_ctx;
+	spdk_nvmf_subsystem_connect_fn		connect_cb;
+	spdk_nvmf_subsystem_disconnect_fn	disconnect_cb;
+
+	TAILQ_HEAD(, spdk_nvmf_session)		sessions;
+	uint32_t				session_id;
+
+	TAILQ_HEAD(, spdk_nvmf_listen_addr)	listen_addrs;
+	uint32_t				num_listen_addrs;
+
+	TAILQ_HEAD(, spdk_nvmf_host)		hosts;
+	uint32_t				num_hosts;
+
+	TAILQ_ENTRY(spdk_nvmf_subsystem) entries;
+};
+
+struct spdk_nvmf_subsystem *spdk_nvmf_create_subsystem(int num,
+		const char *name,
+		enum spdk_nvmf_subtype subtype,
+		enum spdk_nvmf_subsystem_mode mode,
+		void *cb_ctx,
+		spdk_nvmf_subsystem_connect_fn connect_cb,
+		spdk_nvmf_subsystem_disconnect_fn disconnect_cb);
+
+void spdk_nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem);
+
+struct spdk_nvmf_subsystem *
+nvmf_find_subsystem(const char *subnqn, const char *hostnqn);
+
+int
+spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem,
+				 const struct spdk_nvmf_transport *transport,
+				 char *traddr, char *trsvcid);
+
+int
+spdk_nvmf_subsystem_add_host(struct spdk_nvmf_subsystem *subsystem,
+			     char *host_nqn);
+
+int
+nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem,
+			 struct spdk_nvme_ctrlr *ctrlr, struct spdk_pci_device *dev);
+
+void spdk_nvmf_subsystem_poll(struct spdk_nvmf_subsystem *subsystem);
+
+int
+spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bdev *bdev);
+
+int spdk_nvmf_subsystem_set_sn(struct spdk_nvmf_subsystem *subsystem, const char *sn);
+
+struct spdk_nvmf_transport {
+	/**
+	 * Name of the transport.
+	 */
+	const char *name;
+
+	/**
+	 * Initialize the transport.
+	 */
+	int (*transport_init)(uint16_t max_queue_depth, uint32_t max_io_size,
+			      uint32_t in_capsule_data_size);
+
+	/**
+	 * Shut down the transport.
+	 */
+	int (*transport_fini)(void);
+
+	/**
+	 * Check for new connections on the transport.
+	 */
+	void (*acceptor_poll)(void);
+
+	/**
+	  * Instruct the acceptor to listen on the address provided. This
+	  * may be called multiple times.
+	  */
+	int (*listen_addr_add)(struct spdk_nvmf_listen_addr *listen_addr);
+
+	/**
+	 * Fill out a discovery log entry for a specific listen address.
+	 */
+	void (*listen_addr_discover)(struct spdk_nvmf_listen_addr *listen_addr,
+				     struct spdk_nvmf_discovery_log_page_entry *entry);
+
+	/**
+	 * Initialize the transport for the given session
+	 */
+	int (*session_init)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
+
+	/**
+	 * Deinitiallize the transport for the given session
+	 */
+	void (*session_fini)(struct spdk_nvmf_session *session);
+
+	/*
+	 * Signal request completion, which sends a response
+	 * to the originator. A request can either
+	 * be completed or released, but not both.
+	 */
+	int (*req_complete)(struct spdk_nvmf_request *req);
+
+	/*
+	 * Signal that the request can be released without sending
+	 * a response. A request can either be completed or release,
+	 * but not both.
+	 */
+	int (*req_release)(struct spdk_nvmf_request *req);
+
+	/*
+	 * Deinitialize a connection.
+	 */
+	void (*conn_fini)(struct spdk_nvmf_conn *conn);
+
+	/*
+	 * Poll a connection for events.
+	 */
+	int (*conn_poll)(struct spdk_nvmf_conn *conn);
+};
+
+int spdk_nvmf_transport_init(void);
+int spdk_nvmf_transport_fini(void);
+const struct spdk_nvmf_transport *spdk_nvmf_transport_get(const char *name);
+void spdk_nvmf_acceptor_poll(void);
+
+void spdk_nvmf_handle_connect(struct spdk_nvmf_request *req);
+
+void
+spdk_nvmf_session_disconnect(struct spdk_nvmf_conn *conn);
+
+#endif

lib/nvmf/nvmf.c

@@ -35,9 +35,11 @@
 
 #include "spdk/log.h"
 #include "spdk/conf.h"
+#include "spdk/nvmf.h"
+#include "spdk/trace.h"
+
 #include "subsystem.h"
 #include "transport.h"
-#include "spdk/trace.h"
 
 SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF)
 

lib/nvmf/nvmf_internal.h

@@ -43,11 +43,7 @@
 #include "spdk/assert.h"
 #include "spdk/queue.h"
 
-#define nvmf_min(a,b) (((a)<(b))?(a):(b))
-#define nvmf_max(a,b) (((a)>(b))?(a):(b))
-
 #define SPDK_NVMF_DEFAULT_NUM_SESSIONS_PER_LCORE 1
-#define SPDK_NVMF_DEFAULT_SIN_PORT ((uint16_t)4420)
 
 #define OBJECT_NVMF_IO				0x30
 
@@ -70,9 +66,6 @@ struct spdk_nvmf_globals {
 	uint32_t max_io_size;
 };
 
-int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess,
-		  uint32_t in_capsule_data_size, uint32_t max_io_size);
-
 static inline uint32_t
 nvmf_u32log2(uint32_t x)
 {

lib/nvmf/rdma.c

@@ -47,6 +47,7 @@
 #include "transport.h"
 #include "spdk/assert.h"
 #include "spdk/log.h"
+#include "spdk/nvmf.h"
 #include "spdk/nvmf_spec.h"
 #include "spdk/string.h"
 #include "spdk/trace.h"

lib/nvmf/request.h

@@ -73,6 +73,4 @@ spdk_nvmf_request_exec(struct spdk_nvmf_request *req);
 
 int spdk_nvmf_request_complete(struct spdk_nvmf_request *req);
 
-void spdk_nvmf_handle_connect(struct spdk_nvmf_request *req);
-
 #endif

lib/nvmf/session.h

@@ -96,9 +96,6 @@ void spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn,
 			       struct spdk_nvmf_fabric_connect_data *data,
 			       struct spdk_nvmf_fabric_connect_rsp *rsp);
 
-void
-spdk_nvmf_session_disconnect(struct spdk_nvmf_conn *conn);
-
 void
 spdk_nvmf_property_get(struct spdk_nvmf_session *session,
 		       struct spdk_nvmf_fabric_prop_get_cmd *cmd,

lib/nvmf/subsystem.c

@@ -38,6 +38,7 @@
 #include "session.h"
 #include "subsystem.h"
 #include "transport.h"
+
 #include "spdk/log.h"
 #include "spdk/string.h"
 #include "spdk/trace.h"
@@ -117,7 +118,9 @@ spdk_nvmf_valid_nqn(const char *nqn)
 
 struct spdk_nvmf_subsystem *
 spdk_nvmf_create_subsystem(int num, const char *name,
-			   enum spdk_nvmf_subtype subtype, void *cb_ctx,
+			   enum spdk_nvmf_subtype subtype,
+			   enum spdk_nvmf_subsystem_mode mode,
+			   void *cb_ctx,
 			   spdk_nvmf_subsystem_connect_fn connect_cb,
 			   spdk_nvmf_subsystem_disconnect_fn disconnect_cb)
 {
@@ -134,6 +137,7 @@ spdk_nvmf_create_subsystem(int num, const char *name,
 
 	subsystem->num = num;
 	subsystem->subtype = subtype;
+	subsystem->mode = mode;
 	subsystem->cb_ctx = cb_ctx;
 	subsystem->connect_cb = connect_cb;
 	subsystem->disconnect_cb = disconnect_cb;
@@ -142,6 +146,12 @@ spdk_nvmf_create_subsystem(int num, const char *name,
 	TAILQ_INIT(&subsystem->hosts);
 	TAILQ_INIT(&subsystem->sessions);
 
+	if (mode == NVMF_SUBSYSTEM_MODE_DIRECT) {
+		subsystem->ops = &spdk_nvmf_direct_ctrlr_ops;
+	} else {
+		subsystem->ops = &spdk_nvmf_virtual_ctrlr_ops;
+	}
+
 	TAILQ_INSERT_HEAD(&g_subsystems, subsystem, entries);
 
 	return subsystem;
@@ -179,7 +189,7 @@ spdk_nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem)
 		spdk_nvmf_session_destruct(session);
 	}
 
-	if (subsystem->ops) {
+	if (subsystem->ops->detach) {
 		subsystem->ops->detach(subsystem);
 	}
 
@@ -261,7 +271,6 @@ nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem,
 		SPDK_ERRLOG("spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
 		return -1;
 	}
-	subsystem->ops = &spdk_nvmf_direct_ctrlr_ops;
 	return 0;
 }
 
@@ -318,3 +327,15 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
 	subsystem->dev.virt.ns_count++;
 	return 0;
 }
+
+int
+spdk_nvmf_subsystem_set_sn(struct spdk_nvmf_subsystem *subsystem, const char *sn)
+{
+	if (subsystem->mode != NVMF_SUBSYSTEM_MODE_VIRTUAL) {
+		return -1;
+	}
+
+	snprintf(subsystem->dev.virt.sn, sizeof(subsystem->dev.virt.sn), "%s", sn);
+
+	return 0;
+}

lib/nvmf/subsystem.h

@@ -37,141 +37,11 @@
 #include "nvmf_internal.h"
 
 #include "spdk/nvme.h"
-#include "spdk/queue.h"
+#include "spdk/nvmf.h"
-#include "spdk/bdev.h"
-
-struct spdk_nvmf_conn;
-struct spdk_nvmf_subsystem;
-struct spdk_nvmf_request;
-struct spdk_nvmf_session;
-
-#define MAX_VIRTUAL_NAMESPACE 16
-#define MAX_SN_LEN 20
-
-enum spdk_nvmf_subsystem_mode {
-	NVMF_SUBSYSTEM_MODE_DIRECT	= 0,
-	NVMF_SUBSYSTEM_MODE_VIRTUAL	= 1,
-};
-
-struct spdk_nvmf_listen_addr {
-	char					*traddr;
-	char					*trsvcid;
-	const struct spdk_nvmf_transport	*transport;
-	TAILQ_ENTRY(spdk_nvmf_listen_addr)	link;
-};
-
-struct spdk_nvmf_host {
-	char				*nqn;
-	TAILQ_ENTRY(spdk_nvmf_host)	link;
-};
-
-struct spdk_nvmf_ctrlr_ops {
-	/**
-	 * Get NVMe identify controller data.
-	 */
-	void (*ctrlr_get_data)(struct spdk_nvmf_session *session);
-
-	/**
-	 * Process admin command.
-	 */
-	int (*process_admin_cmd)(struct spdk_nvmf_request *req);
-
-	/**
-	 * Process IO command.
-	 */
-	int (*process_io_cmd)(struct spdk_nvmf_request *req);
-
-	/**
-	 * Poll for completions.
-	 */
-	void (*poll_for_completions)(struct spdk_nvmf_session *session);
-
-	/**
-	 * Detach the controller.
-	 */
-	void (*detach)(struct spdk_nvmf_subsystem *subsystem);
-};
-
-typedef void (*spdk_nvmf_subsystem_connect_fn)(void *cb_ctx, struct spdk_nvmf_request *req);
-typedef void (*spdk_nvmf_subsystem_disconnect_fn)(void *cb_ctx, struct spdk_nvmf_conn *conn);
-
-/*
- * The NVMf subsystem, as indicated in the specification, is a collection
- * of virtual controller sessions.  Any individual controller session has
- * access to all the NVMe device/namespaces maintained by the subsystem.
- */
-struct spdk_nvmf_subsystem {
-	uint16_t num;
-	uint32_t lcore;
-	char subnqn[SPDK_NVMF_NQN_MAX_LEN];
-	enum spdk_nvmf_subsystem_mode mode;
-	enum spdk_nvmf_subtype subtype;
-
-	union {
-		struct {
-			struct spdk_nvme_ctrlr *ctrlr;
-			struct spdk_nvme_qpair *io_qpair;
-			struct spdk_pci_device *pci_dev;
-		} direct;
-
-		struct {
-			char	sn[MAX_SN_LEN + 1];
-			struct spdk_bdev *ns_list[MAX_VIRTUAL_NAMESPACE];
-			struct spdk_io_channel *ch[MAX_VIRTUAL_NAMESPACE];
-			uint16_t ns_count;
-		} virt;
-	} dev;
-
-	const struct spdk_nvmf_ctrlr_ops *ops;
-
-	void					*cb_ctx;
-	spdk_nvmf_subsystem_connect_fn		connect_cb;
-	spdk_nvmf_subsystem_disconnect_fn	disconnect_cb;
-
-	TAILQ_HEAD(, spdk_nvmf_session)		sessions;
-	uint32_t				session_id;
-
-	TAILQ_HEAD(, spdk_nvmf_listen_addr)	listen_addrs;
-	uint32_t				num_listen_addrs;
-
-	TAILQ_HEAD(, spdk_nvmf_host)		hosts;
-	uint32_t				num_hosts;
-
-	TAILQ_ENTRY(spdk_nvmf_subsystem) entries;
-};
-
-struct spdk_nvmf_subsystem *spdk_nvmf_create_subsystem(int num,
-		const char *name,
-		enum spdk_nvmf_subtype subtype,
-		void *cb_ctx,
-		spdk_nvmf_subsystem_connect_fn connect_cb,
-		spdk_nvmf_subsystem_disconnect_fn disconnect_cb);
-
-void spdk_nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem);
-
-struct spdk_nvmf_subsystem *
-nvmf_find_subsystem(const char *subnqn, const char *hostnqn);
-
-int
-spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem,
-				 const struct spdk_nvmf_transport *transport,
-				 char *traddr, char *trsvcid);
-
-int
-spdk_nvmf_subsystem_add_host(struct spdk_nvmf_subsystem *subsystem,
-			     char *host_nqn);
-
-int
-nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem,
-			 struct spdk_nvme_ctrlr *ctrlr, struct spdk_pci_device *dev);
 
 void
 spdk_format_discovery_log(struct spdk_nvmf_discovery_log_page *disc_log, uint32_t length);
 
-void spdk_nvmf_subsystem_poll(struct spdk_nvmf_subsystem *subsystem);
-
-int
-spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bdev *bdev);
 extern const struct spdk_nvmf_ctrlr_ops spdk_nvmf_direct_ctrlr_ops;
 extern const struct spdk_nvmf_ctrlr_ops spdk_nvmf_virtual_ctrlr_ops;
 #endif /* SPDK_NVMF_SUBSYSTEM_H */

lib/nvmf/transport.c

@@ -37,6 +37,7 @@
 #include <strings.h>
 
 #include "spdk/log.h"
+#include "spdk/nvmf.h"
 #include "spdk/queue.h"
 
 #include "nvmf_internal.h"

lib/nvmf/transport.h

@@ -34,90 +34,6 @@
 #ifndef SPDK_NVMF_TRANSPORT_H
 #define SPDK_NVMF_TRANSPORT_H
 
-#include <stdint.h>
-
-struct spdk_nvmf_session;
-struct spdk_nvmf_conn;
-struct spdk_nvmf_request;
-struct spdk_nvmf_discovery_log_page_entry;
-struct spdk_nvmf_listen_addr;
-struct spdk_nvmf_request;
-
-struct spdk_nvmf_transport {
-	/**
-	 * Name of the transport.
-	 */
-	const char *name;
-
-	/**
-	 * Initialize the transport.
-	 */
-	int (*transport_init)(uint16_t max_queue_depth, uint32_t max_io_size,
-			      uint32_t in_capsule_data_size);
-
-	/**
-	 * Shut down the transport.
-	 */
-	int (*transport_fini)(void);
-
-	/**
-	 * Check for new connections on the transport.
-	 */
-	void (*acceptor_poll)(void);
-
-	/**
-	  * Instruct the acceptor to listen on the address provided. This
-	  * may be called multiple times.
-	  */
-	int (*listen_addr_add)(struct spdk_nvmf_listen_addr *listen_addr);
-
-	/**
-	 * Fill out a discovery log entry for a specific listen address.
-	 */
-	void (*listen_addr_discover)(struct spdk_nvmf_listen_addr *listen_addr,
-				     struct spdk_nvmf_discovery_log_page_entry *entry);
-
-	/**
-	 * Initialize the transport for the given session
-	 */
-	int (*session_init)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
-
-	/**
-	 * Deinitiallize the transport for the given session
-	 */
-	void (*session_fini)(struct spdk_nvmf_session *session);
-
-	/*
-	 * Signal request completion, which sends a response
-	 * to the originator. A request can either
-	 * be completed or released, but not both.
-	 */
-	int (*req_complete)(struct spdk_nvmf_request *req);
-
-	/*
-	 * Signal that the request can be released without sending
-	 * a response. A request can either be completed or release,
-	 * but not both.
-	 */
-	int (*req_release)(struct spdk_nvmf_request *req);
-
-	/*
-	 * Deinitialize a connection.
-	 */
-	void (*conn_fini)(struct spdk_nvmf_conn *conn);
-
-	/*
-	 * Poll a connection for events.
-	 */
-	int (*conn_poll)(struct spdk_nvmf_conn *conn);
-};
-
-int spdk_nvmf_transport_init(void);
-int spdk_nvmf_transport_fini(void);
-const struct spdk_nvmf_transport *spdk_nvmf_transport_get(const char *name);
-
-void spdk_nvmf_acceptor_poll(void);
-
 extern const struct spdk_nvmf_transport spdk_nvmf_transport_rdma;
 
 #endif /* SPDK_NVMF_TRANSPORT_H */

lib/nvmf/virtual.c

@@ -37,6 +37,8 @@
 #include "subsystem.h"
 #include "session.h"
 #include "request.h"
+
+#include "spdk/bdev.h"
 #include "spdk/endian.h"
 #include "spdk/log.h"
 #include "spdk/nvme.h"

test/lib/nvmf/request/request_ut.c

@@ -47,17 +47,6 @@ void spdk_trace_record(uint16_t tpoint_id, uint16_t poller_id, uint32_t size,
 {
 }
 
-spdk_event_t
-spdk_event_allocate(uint32_t lcore, spdk_event_fn fn, void *arg1, void *arg2, spdk_event_t next)
-{
-	return NULL;
-}
-
-void
-spdk_event_call(spdk_event_t event)
-{
-}
-
 void
 spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn,
 			  struct spdk_nvmf_fabric_connect_cmd *cmd,

test/lib/nvmf/subsystem/subsystem_ut.c

@@ -40,6 +40,7 @@
 #include "subsystem.h"
 
 const struct spdk_nvmf_ctrlr_ops spdk_nvmf_direct_ctrlr_ops;
+const struct spdk_nvmf_ctrlr_ops spdk_nvmf_virtual_ctrlr_ops;
 
 #include "subsystem.c"
 
@@ -47,30 +48,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF)
 
 struct spdk_nvmf_globals g_nvmf_tgt;
 
-uint32_t
-spdk_app_get_current_core(void)
-{
-	return 0;
-}
-
-struct spdk_event *spdk_event_allocate(uint32_t lcore, spdk_event_fn fn,
-				       void *arg1, void *arg2,
-				       spdk_event_t next)
-{
-	return NULL;
-}
-
-void
-spdk_poller_register(struct spdk_poller **ppoller, spdk_poller_fn fn, void *arg, uint32_t lcore,
-		     struct spdk_event *complete, uint64_t period_microseconds)
-{
-}
-
-void spdk_poller_unregister(struct spdk_poller **ppoller,
-			    struct spdk_event *complete)
-{
-}
-
 int32_t
 spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr *ctrlr)
 {
@@ -113,7 +90,8 @@ nvmf_test_create_subsystem(void)
 	struct spdk_nvmf_subsystem *subsystem;
 
 	strncpy(nqn, "nqn.2016-06.io.spdk:subsystem1", sizeof(nqn));
-	subsystem = spdk_nvmf_create_subsystem(1, nqn, SPDK_NVMF_SUBTYPE_NVME, NULL, NULL, NULL);
+	subsystem = spdk_nvmf_create_subsystem(1, nqn, SPDK_NVMF_SUBTYPE_NVME,
+					       NVMF_SUBSYSTEM_MODE_DIRECT, NULL, NULL, NULL);
 	SPDK_CU_ASSERT_FATAL(subsystem != NULL);
 	CU_ASSERT_EQUAL(subsystem->num, 1);
 	CU_ASSERT_STRING_EQUAL(subsystem->subnqn, nqn);
@@ -124,7 +102,8 @@ nvmf_test_create_subsystem(void)
 	memset(nqn + strlen(nqn), 'a', 222 - strlen(nqn));
 	nqn[222] = '\0';
 	CU_ASSERT(strlen(nqn) == 222);
-	subsystem = spdk_nvmf_create_subsystem(2, nqn, SPDK_NVMF_SUBTYPE_NVME, NULL, NULL, NULL);
+	subsystem = spdk_nvmf_create_subsystem(2, nqn, SPDK_NVMF_SUBTYPE_NVME,
+					       NVMF_SUBSYSTEM_MODE_DIRECT, NULL, NULL, NULL);
 	SPDK_CU_ASSERT_FATAL(subsystem != NULL);
 	CU_ASSERT_STRING_EQUAL(subsystem->subnqn, nqn);
 	spdk_nvmf_delete_subsystem(subsystem);
@@ -134,7 +113,8 @@ nvmf_test_create_subsystem(void)
 	memset(nqn + strlen(nqn), 'a', 223 - strlen(nqn));
 	nqn[223] = '\0';
 	CU_ASSERT(strlen(nqn) == 223);
-	subsystem = spdk_nvmf_create_subsystem(2, nqn, SPDK_NVMF_SUBTYPE_NVME, NULL, NULL, NULL);
+	subsystem = spdk_nvmf_create_subsystem(2, nqn, SPDK_NVMF_SUBTYPE_NVME,
+					       NVMF_SUBSYSTEM_MODE_DIRECT, NULL, NULL, NULL);
 	CU_ASSERT(subsystem == NULL);
 }