lib/event: support ABI compatibility for spdk_app_opts.

This patch is used to support ABI compatability issue for spdk_app_opts.

Fixes #1484

Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Change-Id: I6fed777fa15b367f7c3706b8f218d86d56493906
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5330
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jacek Kalwas <jacek.kalwas@intel.com>

CHANGELOG.md

@@ -19,6 +19,10 @@ The pci_whitelist, pci_blacklist and master_core members of struct spdk_env_opts
 have been deprecated.  The new members are named pci_allowed, pci_blocked and
 main_core respectively.
 
+An `opts_size`element was added in the `spdk_app_opts` structure
+to solve the ABI compatiblity issue between different SPDK version. An `opts_size`
+parameter is added into `spdk_app_opts_init` function.
+
 ### nvmf
 
 Broadcom FC LLD driver and SPDK NVMe-oF FC transport consolidated one LLD API,

app/iscsi_tgt/iscsi_tgt.c

@@ -75,7 +75,7 @@ main(int argc, char **argv)
 	int rc;
 	struct spdk_app_opts opts = {};
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "iscsi";
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "b", NULL,
 				      iscsi_parse_arg, iscsi_usage)) !=

app/nvmf_tgt/nvmf_main.c

@@ -63,7 +63,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 
 	/* default value in opts */
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "nvmf";
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "", NULL,
 				      nvmf_parse_arg, nvmf_usage)) !=

app/spdk_dd/spdk_dd.c

@@ -1104,7 +1104,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 	int rc = 1;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "spdk_dd";
 	opts.reactor_mask = "0x1";
 	opts.shutdown_cb = dd_finish;

app/spdk_tgt/spdk_tgt.c

@@ -109,7 +109,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "spdk_tgt";
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, g_spdk_tgt_get_opts_string,
 				      NULL, spdk_tgt_parse_arg, spdk_tgt_usage)) !=

app/vhost/vhost.c

@@ -88,7 +88,7 @@ main(int argc, char *argv[])
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "vhost";
 
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "f:S:", NULL,

examples/accel/perf/accel_perf.c

@@ -676,7 +676,7 @@ main(int argc, char **argv)
 	int rc = 0;
 
 	pthread_mutex_init(&g_workers_lock, NULL);
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.reactor_mask = "0x1";
 	if (spdk_app_parse_args(argc, argv, &opts, "o:q:t:yw:P:f:", NULL, parse_args,
 				usage) != SPDK_APP_PARSE_ARGS_SUCCESS) {

examples/bdev/hello_world/hello_bdev.c

@@ -261,7 +261,7 @@ main(int argc, char **argv)
 	struct hello_context_t hello_context = {};
 
 	/* Set default values in opts structure. */
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "hello_bdev";
 
 	/*

examples/blob/cli/blobcli.c

@@ -1542,7 +1542,7 @@ main(int argc, char **argv)
 	}
 
 	/* Set default values in opts struct along with name and conf file. */
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "blobcli";
 	opts.json_config_file = cli_context->config_file;
 

examples/blob/hello_world/hello_blob.c

@@ -451,7 +451,7 @@ main(int argc, char **argv)
 	SPDK_NOTICELOG("entry\n");
 
 	/* Set default values in opts structure. */
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 
 	/*
 	 * Setup a few specifics before we init, for most SPDK cmd line

examples/interrupt_tgt/interrupt_tgt.c

@@ -70,7 +70,7 @@ main(int argc, char *argv[])
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "interrupt_tgt";
 
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "S:E", NULL,

examples/sock/hello_world/hello_sock.c

@@ -410,7 +410,7 @@ main(int argc, char **argv)
 	struct hello_context_t hello_context = {};
 
 	/* Set default values in opts structure. */
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "hello_sock";
 	opts.shutdown_cb = hello_sock_shutdown_cb;
 

include/spdk/event.h

@@ -149,14 +149,23 @@ struct spdk_app_opts {
 	logfunc         *log;
 
 	uint64_t		base_virtaddr;
+
+	/**
+	 * The size of spdk_app_opts according to the caller of this library is used for ABI
+	 * compatibility. The library uses this field to know how many fields in this
+	 * structure are valid. And the library will populate any remaining fields with default values.
+	 * After that, new added fields should be put after opts_size.
+	 */
+	size_t opts_size;
 };
 
 /**
  * Initialize the default value of opts
  *
  * \param opts Data structure where SPDK will initialize the default options.
+ * \param opts_size Must be set to sizeof(struct spdk_app_opts).
  */
-void spdk_app_opts_init(struct spdk_app_opts *opts);
+void spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size);
 
 /**
  * Start the framework.
@@ -179,14 +188,15 @@ void spdk_app_opts_init(struct spdk_app_opts *opts);
  * condition occurs during the intialization code within spdk_app_start(),
  * this function will immediately return before invoking start_fn.
  *
- * \param opts Initialization options used for this application.
+ * \param opts_user Initialization options used for this application. It should not be
+ *             NULL. And the opts_size value inside the opts structure should not be zero.
  * \param start_fn Entry point that will execute on an internally created thread
  *                 once the framework has been started.
  * \param ctx Argument passed to function start_fn.
  *
  * \return 0 on success or non-zero on failure.
  */
-int spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn,
+int spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn,
 		   void *ctx);
 
 /**

lib/event/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 6
+SO_VER := 7
 SO_MINOR := 0
 
 CFLAGS += $(ENV_CFLAGS)

lib/event/app.c

@@ -36,6 +36,7 @@
 
 #include "spdk_internal/event.h"
 
+#include "spdk/assert.h"
 #include "spdk/env.h"
 #include "spdk/log.h"
 #include "spdk/thread.h"
@@ -186,25 +187,38 @@ app_opts_validate(const char *app_opts)
 }
 
 void
-spdk_app_opts_init(struct spdk_app_opts *opts)
+spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size)
 {
 	if (!opts) {
+		SPDK_ERRLOG("opts should not be NULL\n");
 		return;
 	}
 
-	memset(opts, 0, sizeof(*opts));
+	if (!opts_size) {
+		SPDK_ERRLOG("opts_size should not be zero value\n");
+		return;
+	}
 
-	opts->enable_coredump = true;
+	memset(opts, 0, opts_size);
-	opts->shm_id = -1;
+	opts->opts_size = opts_size;
-	opts->mem_size = SPDK_APP_DPDK_DEFAULT_MEM_SIZE;
+
-	opts->main_core = SPDK_APP_DPDK_DEFAULT_MAIN_CORE;
+#define SET_FIELD(field, value) \
-	opts->mem_channel = SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL;
+	if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= opts_size) { \
-	opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK;
+		opts->field = value; \
-	opts->base_virtaddr = SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR;
+	} \
-	opts->print_level = SPDK_APP_DEFAULT_LOG_PRINT_LEVEL;
+
-	opts->rpc_addr = SPDK_DEFAULT_RPC_ADDR;
+	SET_FIELD(enable_coredump, true);
-	opts->num_entries = SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES;
+	SET_FIELD(shm_id, -1);
-	opts->delay_subsystem_init = false;
+	SET_FIELD(mem_size, SPDK_APP_DPDK_DEFAULT_MEM_SIZE);
+	SET_FIELD(main_core, SPDK_APP_DPDK_DEFAULT_MAIN_CORE);
+	SET_FIELD(mem_channel, SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL);
+	SET_FIELD(reactor_mask, SPDK_APP_DPDK_DEFAULT_CORE_MASK);
+	SET_FIELD(base_virtaddr, SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR);
+	SET_FIELD(print_level, SPDK_APP_DEFAULT_LOG_PRINT_LEVEL);
+	SET_FIELD(rpc_addr, SPDK_DEFAULT_RPC_ADDR);
+	SET_FIELD(num_entries, SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES);
+	SET_FIELD(delay_subsystem_init, false);
+#undef SET_FIELD
 }
 
 static int
@@ -391,20 +405,77 @@ bootstrap_fn(void *arg1)
 	}
 }
 
+static void
+app_copy_opts(struct spdk_app_opts *opts, struct spdk_app_opts *opts_user, size_t opts_size)
+{
+	spdk_app_opts_init(opts, sizeof(*opts));
+	opts->opts_size = opts_size;
+
+#define SET_FIELD(field) \
+        if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= (opts->opts_size)) { \
+		opts->field = opts_user->field; \
+	} \
+
+	SET_FIELD(name);
+	SET_FIELD(config_file);
+	SET_FIELD(json_config_file);
+	SET_FIELD(json_config_ignore_errors);
+	SET_FIELD(rpc_addr);
+	SET_FIELD(reactor_mask);
+	SET_FIELD(tpoint_group_mask);
+	SET_FIELD(shm_id);
+	SET_FIELD(shutdown_cb);
+	SET_FIELD(enable_coredump);
+	SET_FIELD(mem_channel);
+	SET_FIELD(main_core);
+	SET_FIELD(mem_size);
+	SET_FIELD(no_pci);
+	SET_FIELD(hugepage_single_segments);
+	SET_FIELD(unlink_hugepage);
+	SET_FIELD(hugedir);
+	SET_FIELD(print_level);
+	SET_FIELD(num_pci_addr);
+	SET_FIELD(pci_blocked);
+	SET_FIELD(pci_allowed);
+	SET_FIELD(iova_mode);
+	SET_FIELD(max_delay_us);
+	SET_FIELD(delay_subsystem_init);
+	SET_FIELD(num_entries);
+	SET_FIELD(env_context);
+	SET_FIELD(log);
+	SET_FIELD(base_virtaddr);
+
+	/* You should not remove this statement, but need to update the assert statement
+	 * if you add a new field, and also add a corresponding SET_FIELD statement */
+	SPDK_STATIC_ASSERT(sizeof(struct spdk_app_opts) == 200, "Incorrect size");
+
+#undef SET_FIELD
+#undef FIELD_CHECK
+}
+
 int
-spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn,
+spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn,
 	       void *arg1)
 {
 	int			rc;
 	char			*tty;
 	struct spdk_cpuset	tmp_cpumask = {};
 	static bool		g_env_was_setup = false;
+	struct spdk_app_opts opts_local = {};
+	struct spdk_app_opts *opts = &opts_local;
 
-	if (!opts) {
+	if (!opts_user) {
-		SPDK_ERRLOG("opts should not be NULL\n");
+		SPDK_ERRLOG("opts_user should not be NULL\n");
 		return 1;
 	}
 
+	if (!opts_user->opts_size) {
+		SPDK_ERRLOG("The opts_size in opts_user structure should not be zero value\n");
+		return 1;
+	}
+
+	app_copy_opts(opts, opts_user, opts_user->opts_size);
+
 	if (opts->config_file) {
 		SPDK_ERRLOG("opts->config_file is deprecated.  Use opts->json_config_file instead.\n");
 		/* For now we will just treat config_file as json_config_file.  But if both were

lib/rocksdb/env_spdk.cc

@@ -736,7 +736,7 @@ SpdkEnv::SpdkEnv(Env *base_env, const std::string &dir, const std::string &conf,
 {
 	struct spdk_app_opts *opts = new struct spdk_app_opts;
 
-	spdk_app_opts_init(opts);
+	spdk_app_opts_init(opts, sizeof(*opts));
 	opts->name = "rocksdb";
 	opts->json_config_file = mConfig.c_str();
 	opts->shutdown_cb = rocksdb_shutdown;

test/app/bdev_svc/bdev_svc.c

@@ -84,7 +84,7 @@ main(int argc, char **argv)
 	const char *reactor_mask = "0x1";
 
 	/* default value in opts structure */
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 
 	opts.name = "bdev_svc";
 	opts.reactor_mask = reactor_mask;

test/app/fuzz/iscsi_fuzz/iscsi_fuzz.c

@@ -1074,7 +1074,7 @@ main(int argc, char **argv)
 
 	TAILQ_INIT(&g_get_pdu_list);
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "iscsi_fuzz";
 
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "T:S:t:", NULL, iscsi_fuzz_parse,

test/app/fuzz/nvme_fuzz/nvme_fuzz.c

@@ -908,7 +908,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "nvme_fuzz";
 
 	g_runtime = DEFAULT_RUNTIME;

test/app/fuzz/vhost_fuzz/vhost_fuzz.c

@@ -1112,7 +1112,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "vhost_fuzz";
 	g_runtime = DEFAULT_RUNTIME;
 

test/app/stub/stub.c

@@ -150,7 +150,7 @@ main(int argc, char **argv)
 	long int val;
 
 	/* default value in opts structure */
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 
 	opts.name = "stub";
 	opts.rpc_addr = NULL;

test/bdev/bdevio/bdevio.c

@@ -1425,7 +1425,7 @@ main(int argc, char **argv)
 	int			rc;
 	struct spdk_app_opts	opts = {};
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "bdevio";
 	opts.reactor_mask = "0x7";
 	opts.shutdown_cb = spdk_bdevio_shutdown_cb;

test/bdev/bdevperf/bdevperf.c

@@ -2093,7 +2093,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "bdevperf";
 	opts.rpc_addr = NULL;
 	opts.shutdown_cb = spdk_bdevperf_shutdown_cb;

test/blobfs/fuse/fuse.c

@@ -92,7 +92,7 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "spdk_fuse";
 	opts.json_config_file = argv[1];
 	opts.reactor_mask = "0x3";

test/blobfs/mkfs/mkfs.c

@@ -94,7 +94,7 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "spdk_mkfs";
 	opts.json_config_file = argv[1];
 	opts.reactor_mask = "0x3";

test/event/app_repeat/app_repeat.c

@@ -83,7 +83,7 @@ main(int argc, char **argv)
 	int rc;
 	int i;
 
-	spdk_app_opts_init(&g_opts);
+	spdk_app_opts_init(&g_opts, sizeof(g_opts));
 	g_opts.name = "app_repeat";
 	g_opts.shutdown_cb = _app_repeat_shutdown_cb;
 	if ((rc = spdk_app_parse_args(argc, argv, &g_opts, g_app_repeat_get_opts_string,

test/event/event_perf/event_perf.c

@@ -144,6 +144,7 @@ main(int argc, char **argv)
 	int op;
 	int rc = 0;
 
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "event_perf";
 
 	g_time_in_sec = 0;

test/event/reactor/reactor.c

@@ -115,7 +115,7 @@ main(int argc, char **argv)
 	int op;
 	int rc = 0;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "reactor";
 
 	g_time_in_sec = 0;

test/event/reactor_perf/reactor_perf.c

@@ -105,7 +105,7 @@ main(int argc, char **argv)
 	int rc;
 	long int val;
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.name = "reactor_perf";
 
 	g_time_in_sec = 0;