ublk: add configure and event/subsystem

ublk backend could support ublk driver with kernel. Specify
configuration parameter to start it up.

Signed-off-by: Yifan Bian <yifan.bian@intel.com>
Co-authored-by: Xiaodong Liu <xiaodong.liu@intel.com>
Change-Id: I55e7d757e04315b25e9bfab5fdcbb6621be3e29e
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15680
Reviewed-by: Xiaodong Liu <xiaodong.liu@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

CONFIG

@@ -113,6 +113,9 @@ CONFIG_RBD=n
 CONFIG_DAOS=n
 CONFIG_DAOS_DIR=
 
+# Build UBLK support
+CONFIG_UBLK=n
+
 # Build vhost library.
 CONFIG_VHOST=y
 

app/spdk_tgt/Makefile

@@ -21,6 +21,9 @@ endif
 
 ifeq ($(OS),Linux)
 SPDK_LIB_LIST += event_nbd
+ifeq ($(CONFIG_UBLK),y)
+SPDK_LIB_LIST += event_ublk
+endif
 ifeq ($(CONFIG_VHOST),y)
 SPDK_LIB_LIST += event_vhost_blk event_vhost_scsi
 endif

configure

@@ -80,6 +80,8 @@ function usage() {
 	echo " --without-dpdk-compressdev No path required."
 	echo " --with-rbd                Build Ceph RBD bdev module."
 	echo " --without-rbd             No path required."
+	echo " --with-ublk               Build ublk library."
+	echo " --without-ublk            No path required."
 	echo " --with-rdma[=DIR]         Build RDMA transport for NVMf target and initiator."
 	echo " --without-rdma            Accepts optional RDMA provider name. Can be \"verbs\" or \"mlx5_dv\"."
 	echo "                           If no provider specified, \"verbs\" provider is used by default."
@@ -436,6 +438,12 @@ for i in "$@"; do
 		--with-env=*)
 			CONFIG[ENV]="${i#*=}"
 			;;
+		--with-ublk)
+			CONFIG[UBLK]=y
+			;;
+		--without-ublk)
+			CONFIG[UBLK]=n
+			;;
 		--with-rbd)
 			CONFIG[RBD]=y
 			;;
@@ -1025,6 +1033,16 @@ if [[ "${CONFIG[RBD]}" = "y" ]]; then
 	fi
 fi
 
+if [[ "${CONFIG[UBLK]}" = "y" ]]; then
+	if ! echo -e '#include <linux/ublk_cmd.h>\n#include <liburing.h>\n' \
+		'int main(void) { return 0; }\n' \
+		| "${BUILD_CMD[@]}" -luring - 2> /dev/null; then
+		echo "--with-ublk requires liburing and ublk_drv."
+		echo "Please install then re-run this script."
+		exit 1
+	fi
+fi
+
 if [[ "${CONFIG[ISCSI_INITIATOR]}" = "y" ]]; then
 	# Fedora installs libiscsi to /usr/lib64/iscsi for some reason.
 	if ! echo -e '#include <iscsi/iscsi.h>\n#include <iscsi/scsi-lowlevel.h>\n' \

include/spdk/ublk.h

@@ -0,0 +1,42 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#ifndef SPDK_UBLK_H_
+#define SPDK_UBLK_H_
+
+#include "spdk/json.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void (*spdk_ublk_fini_cb)(void *arg);
+
+/**
+ * Initialize the ublk library.
+ */
+void spdk_ublk_init(void);
+
+/**
+ * Stop the ublk layer and close all running ublk block devices.
+ *
+ * \param cb_fn Callback to be always called.
+ * \param cb_arg Passed to cb_fn.
+ * \return 0 on success.
+ */
+int spdk_ublk_fini(spdk_ublk_fini_cb cb_fn, void *cb_arg);
+
+/**
+ * Write UBLK subsystem configuration into provided JSON context.
+ *
+ * \param w JSON write context
+ */
+void spdk_ublk_write_config_json(struct spdk_json_write_ctx *w);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SPDK_UBLK_H_ */

lib/Makefile

@@ -12,6 +12,9 @@ DIRS-y += bdev blob blobfs conf dma accel event json jsonrpc \
           ioat ut_mock iscsi notify init trace_parser
 ifeq ($(OS),Linux)
 DIRS-y += nbd ftl vfio_user
+ifeq ($(CONFIG_UBLK),y)
+DIRS-y += ublk
+endif
 endif
 
 DIRS-$(CONFIG_OCF) += env_ocf

lib/ublk/Makefile

@@ -0,0 +1,18 @@
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (C) 2022 Intel Corporation.
+#  All rights reserved.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+SO_VER := 1
+SO_MINOR := 0
+
+C_SRCS = ublk.c
+LIBNAME = ublk
+LOCAL_SYS_LIBS = -luring
+
+SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ublk.map)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/ublk/spdk_ublk.map

@@ -0,0 +1,9 @@
+{
+	global:
+
+	spdk_ublk_init;
+	spdk_ublk_fini;
+	spdk_ublk_write_config_json;
+
+	local: *;
+};

lib/ublk/ublk.c

@@ -0,0 +1,46 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include <linux/ublk_cmd.h>
+#include <liburing.h>
+
+#include "spdk/env.h"
+#include "spdk/json.h"
+#include "spdk/ublk.h"
+#include "spdk/thread.h"
+
+struct ublk_tgt {
+
+	spdk_ublk_fini_cb fini_cb_fn;
+	void *fini_cb_arg;
+};
+
+static TAILQ_HEAD(, spdk_ublk_dev) g_ublk_bdevs = TAILQ_HEAD_INITIALIZER(g_ublk_bdevs);
+static struct ublk_tgt g_ublk_tgt;
+
+void
+spdk_ublk_init(void)
+{
+	assert(spdk_get_thread() == spdk_thread_get_app_thread());
+}
+
+int
+spdk_ublk_fini(spdk_ublk_fini_cb cb_fn, void *cb_arg)
+{
+	assert(spdk_get_thread() == spdk_thread_get_app_thread());
+	g_ublk_tgt.fini_cb_fn = cb_fn;
+	g_ublk_tgt.fini_cb_arg = cb_arg;
+	g_ublk_tgt.fini_cb_fn(g_ublk_tgt.fini_cb_arg);
+
+	return 0;
+}
+
+void
+spdk_ublk_write_config_json(struct spdk_json_write_ctx *w)
+{
+	spdk_json_write_array_begin(w);
+
+	spdk_json_write_array_end(w);
+}

mk/spdk.common.mk

@@ -240,6 +240,10 @@ LDFLAGS += -L$(CONFIG_DAOS_DIR)/lib64
 endif
 endif
 
+ifeq ($(CONFIG_UBLK),y)
+SYS_LIBS += -luring
+endif
+
 #Attach only if FreeBSD and RDMA is specified with configure
 ifeq ($(OS),FreeBSD)
 ifeq ($(CONFIG_RDMA),y)

mk/spdk.lib_deps.mk

@@ -66,6 +66,9 @@ DEPDIRS-init := jsonrpc json log rpc thread util
 
 DEPDIRS-ftl := log util thread bdev trace
 DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev
+ifeq ($(CONFIG_UBLK),y)
+DEPDIRS-ublk := log util thread $(JSON_LIBS) bdev
+endif
 DEPDIRS-nvmf := accel log sock util nvme thread $(JSON_LIBS) trace bdev
 ifeq ($(CONFIG_RDMA),y)
 DEPDIRS-nvmf += rdma
@@ -164,6 +167,9 @@ DEPDIRS-event_bdev := init bdev event_accel event_vmd event_sock event_iobuf
 DEPDIRS-event_scheduler := event init json log
 
 DEPDIRS-event_nbd := init nbd event_bdev
+ifeq ($(CONFIG_UBLK),y)
+DEPDIRS-event_ublk := init ublk event_bdev
+endif
 DEPDIRS-event_nvmf := init nvmf event_bdev event_scheduler event_sock thread log bdev util $(JSON_LIBS)
 DEPDIRS-event_scsi := init scsi event_bdev
 

module/event/subsystems/Makefile

@@ -10,6 +10,9 @@ DIRS-y += bdev accel scheduler iscsi nvmf scsi vmd sock iobuf
 
 ifeq ($(OS),Linux)
 DIRS-y += nbd
+ifeq ($(CONFIG_UBLK),y)
+DIRS-y += ublk
+endif
 endif
 
 DIRS-$(CONFIG_VHOST) += vhost_blk vhost_scsi
@@ -23,6 +26,7 @@ DEPDIRS-accel := iobuf
 DEPDIRS-bdev := accel vmd sock iobuf
 DEPDIRS-iscsi := scsi
 DEPDIRS-nbd := bdev
+DEPDIRS-ublk := bdev
 DEPDIRS-nvmf := bdev
 DEPDIRS-scsi := bdev
 DEPDIRS-vhost_scsi := scsi

module/event/subsystems/ublk/Makefile

@@ -0,0 +1,17 @@
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (C) 2022 Intel Corporation.
+#  All rights reserved.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+SO_VER := 1
+SO_MINOR := 0
+
+C_SRCS = ublk.c
+LIBNAME = event_ublk
+
+SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

module/event/subsystems/ublk/ublk.c

@@ -0,0 +1,48 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/ublk.h"
+#include "spdk_internal/init.h"
+
+static void
+ublk_subsystem_init(void)
+{
+	spdk_ublk_init();
+	spdk_subsystem_init_next(0);
+}
+
+static void
+ublk_subsystem_fini_done(void *arg)
+{
+	spdk_subsystem_fini_next();
+}
+
+static void
+ublk_subsystem_fini(void)
+{
+	int rc;
+
+	rc = spdk_ublk_fini(ublk_subsystem_fini_done, NULL);
+	if (rc != 0) {
+		ublk_subsystem_fini_done(NULL);
+	}
+}
+
+static void
+ublk_subsystem_write_config_json(struct spdk_json_write_ctx *w)
+{
+	spdk_ublk_write_config_json(w);
+}
+
+static struct spdk_subsystem g_spdk_subsystem_ublk = {
+	.name = "ublk",
+	.init = ublk_subsystem_init,
+	.fini = ublk_subsystem_fini,
+	.write_config_json = ublk_subsystem_write_config_json,
+};
+
+SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_ublk);
+SPDK_SUBSYSTEM_DEPEND(ublk, bdev)

test/common/skipped_build_files.txt

@@ -61,3 +61,7 @@ module/bdev/xnvme/bdev_xnvme_rpc
 module/accel/mlx5/accel_mlx5
 module/accel/mlx5/accel_mlx5_rpc
 lib/mlx5/mlx5_crypto
+
+# Not configured to test ublk
+lib/ublk/ublk
+module/event/subsystems/ublk/ublk