module/iobuf: introduce iobuf module

This gives us a place to initialize iobuf pools, specify subsystem
dependencies, and execute RPCs to configure the sizes of the pools.

We allow users to configure the size of the pools either through the
options in spdk_bdev_opts or through the new RPC, iobuf_set_options.
The second option has higher priority, so it will override the options
set by the bdev layer.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I7c45ace04bc71c8343658f98248fc7babcf24e5d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15765
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

doc/jsonrpc.md

@@ -11150,3 +11150,42 @@ Example response:
   "result": true
 }
 ~~~
+
+### iobuf_set_options {#rpc_iobuf_set_options}
+
+Set iobuf buffer pool options.
+
+#### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+small_pool_count        | Optional | number      | Number of small buffers in the global pool
+large_pool_count        | Optional | number      | Number of large buffers in the global pool
+small_bufsize           | Optional | number      | Size of a small buffer
+large_bufsize           | Optional | number      | Size of a small buffer
+
+#### Example
+
+Example request:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "iobuf_set_options",
+  "params": {
+    "small_pool_count": 16383,
+    "large_pool_count": 2047
+  }
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~

mk/spdk.lib_deps.mk

@@ -162,6 +162,7 @@ DEPDIRS-event_vhost_blk := init vhost
 DEPDIRS-event_vhost_scsi := init vhost event_scheduler event_scsi
 DEPDIRS-event_sock := init sock
 DEPDIRS-event_vfu_tgt := init vfu_tgt
+DEPDIRS-event_iobuf := init log thread util $(JSON_LIBS)
 
 # module/vfu_device
 

module/event/subsystems/Makefile

@@ -6,7 +6,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y += bdev accel scheduler iscsi nvmf scsi vmd sock
+DIRS-y += bdev accel scheduler iscsi nvmf scsi vmd sock iobuf
 
 ifeq ($(OS),Linux)
 DIRS-y += nbd

module/event/subsystems/iobuf/Makefile

@@ -0,0 +1,17 @@
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (C) 2022 Intel Corporation.
+#  All rights reserved.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+SO_VER := 1
+SO_MINOR := 0
+
+C_SRCS = iobuf.c iobuf_rpc.c
+LIBNAME = event_iobuf
+
+SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

module/event/subsystems/iobuf/iobuf.c

@@ -0,0 +1,106 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/bdev.h"
+#include "spdk/log.h"
+#include "spdk/thread.h"
+#include "spdk_internal/init.h"
+
+int iobuf_set_opts(struct spdk_iobuf_opts *opts);
+
+static struct spdk_iobuf_opts g_opts;
+static bool g_opts_set;
+
+int
+iobuf_set_opts(struct spdk_iobuf_opts *opts)
+{
+	int rc;
+
+	rc = spdk_iobuf_set_opts(opts);
+	if (rc != 0) {
+		return rc;
+	}
+
+	g_opts = *opts;
+	g_opts_set = true;
+
+	return 0;
+}
+
+static void
+iobuf_subsystem_initialize(void)
+{
+	int rc;
+
+	if (g_opts_set) {
+		/* We want to allow users to keep using bdev layer's spdk_bdev_opts to specify the
+		 * sizes of the pools, but want to have iobuf_set_opts to take precedence over what
+		 * was set by the spdk_bdev_opts.  So, reset the opts here in case bdev layer set
+		 * them after iobuf_set_opts.
+		 */
+		rc = spdk_iobuf_set_opts(&g_opts);
+		if (rc != 0) {
+			/* This should never happen, we've already validated these options */
+			assert(0);
+			goto finish;
+		}
+	}
+
+	rc = spdk_iobuf_initialize();
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to initialize iobuf\n");
+	}
+finish:
+	spdk_subsystem_init_next(rc);
+}
+
+static void
+iobuf_finish_cb(void *ctx)
+{
+	spdk_subsystem_fini_next();
+}
+
+static void
+iobuf_subsystem_finish(void)
+{
+	spdk_iobuf_finish(iobuf_finish_cb, NULL);
+}
+
+static void
+iobuf_write_config_json(struct spdk_json_write_ctx *w)
+{
+	struct spdk_iobuf_opts opts;
+
+	spdk_iobuf_get_opts(&opts);
+
+	spdk_json_write_array_begin(w);
+	/* Make sure we don't override the options from spdk_bdev_opts, unless iobuf_set_options
+	 * has been executed
+	 */
+	if (g_opts_set) {
+		spdk_json_write_object_begin(w);
+		spdk_json_write_named_string(w, "method", "iobuf_set_options");
+
+		spdk_json_write_named_object_begin(w, "params");
+		spdk_json_write_named_uint64(w, "small_pool_count", opts.small_pool_count);
+		spdk_json_write_named_uint64(w, "large_pool_count", opts.large_pool_count);
+		spdk_json_write_named_uint32(w, "small_bufsize", opts.small_bufsize);
+		spdk_json_write_named_uint32(w, "large_bufsize", opts.large_bufsize);
+		spdk_json_write_object_end(w);
+
+		spdk_json_write_object_end(w);
+	}
+	spdk_json_write_array_end(w);
+}
+
+static struct spdk_subsystem g_subsystem_iobuf = {
+	.name = "iobuf",
+	.init = iobuf_subsystem_initialize,
+	.fini = iobuf_subsystem_finish,
+	.write_config_json = iobuf_write_config_json,
+};
+
+SPDK_SUBSYSTEM_REGISTER(g_subsystem_iobuf);

module/event/subsystems/iobuf/iobuf_rpc.c

@@ -0,0 +1,44 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/thread.h"
+#include "spdk/rpc.h"
+#include "spdk/string.h"
+#include "spdk_internal/init.h"
+
+int iobuf_set_opts(struct spdk_iobuf_opts *opts);
+
+static const struct spdk_json_object_decoder rpc_iobuf_set_options_decoders[] = {
+	{"small_pool_count", offsetof(struct spdk_iobuf_opts, small_pool_count), spdk_json_decode_uint64, true},
+	{"large_pool_count", offsetof(struct spdk_iobuf_opts, large_pool_count), spdk_json_decode_uint64, true},
+	{"small_bufsize", offsetof(struct spdk_iobuf_opts, small_bufsize), spdk_json_decode_uint32, true},
+	{"large_bufsize", offsetof(struct spdk_iobuf_opts, large_bufsize), spdk_json_decode_uint32, true},
+};
+
+static void
+rpc_iobuf_set_options(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params)
+{
+	struct spdk_iobuf_opts opts;
+	int rc;
+
+	spdk_iobuf_get_opts(&opts);
+	rc = spdk_json_decode_object(params, rpc_iobuf_set_options_decoders,
+				     SPDK_COUNTOF(rpc_iobuf_set_options_decoders), &opts);
+	if (rc != 0) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "spdk_json_decode_object failed");
+		return;
+	}
+
+	rc = iobuf_set_opts(&opts);
+	if (rc != 0) {
+		spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
+		return;
+	}
+
+	spdk_jsonrpc_send_bool_response(request, true);
+}
+SPDK_RPC_REGISTER("iobuf_set_options", rpc_iobuf_set_options, SPDK_RPC_STARTUP)

python/spdk/rpc/__init__.py

@@ -30,6 +30,7 @@ from . import vhost
 from . import vmd
 from . import sock
 from . import vfio_user
+from . import iobuf
 from . import client as rpc_client
 
 

python/spdk/rpc/iobuf.py

@@ -0,0 +1,25 @@
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (C) 2022 Intel Corporation.
+#  All rights reserved.
+
+def iobuf_set_options(client, small_pool_count, large_pool_count, small_bufsize, large_bufsize):
+    """Set iobuf pool options.
+
+    Args:
+        small_pool_count: number of small buffers in the global pool
+        large_pool_count: number of large buffers in the global pool
+        small_bufsize: size of a small buffer
+        large_bufsize: size of a large buffer
+    """
+    params = {}
+
+    if small_pool_count is not None:
+        params['small_pool_count'] = small_pool_count
+    if large_pool_count is not None:
+        params['large_pool_count'] = large_pool_count
+    if small_bufsize is not None:
+        params['small_bufsize'] = small_bufsize
+    if large_bufsize is not None:
+        params['large_bufsize'] = large_bufsize
+
+    return client.call('iobuf_set_options', params)

scripts/rpc.py

@@ -3192,6 +3192,19 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
     p.add_argument('new_size', help='new bdev size for resize operation. The unit is MiB')
     p.set_defaults(func=bdev_daos_resize)
 
+    def iobuf_set_options(args):
+        rpc.iobuf.iobuf_set_options(args.client,
+                                    small_pool_count=args.small_pool_count,
+                                    large_pool_count=args.large_pool_count,
+                                    small_bufsize=args.small_bufsize,
+                                    large_bufsize=args.large_bufsize)
+    p = subparsers.add_parser('iobuf_set_options', help='Set iobuf pool options')
+    p.add_argument('--small-pool-count', help='number of small buffers in the global pool', type=int)
+    p.add_argument('--large-pool-count', help='number of large buffers in the global pool', type=int)
+    p.add_argument('--small-bufsize', help='size of a small buffer', type=int)
+    p.add_argument('--large-bufsize', help='size of a large buffer', type=int)
+    p.set_defaults(func=iobuf_set_options)
+
     def check_called_name(name):
         if name in deprecated_aliases:
             print("{} is deprecated, use {} instead.".format(name, deprecated_aliases[name]), file=sys.stderr)

test/json_config/clear_config.py

@@ -175,6 +175,10 @@ def clear_scheduler_subsystem(args, scheduler_config):
     pass
 
 
+def clear_iobuf_subsystem(args, config):
+    pass
+
+
 def call_test_cmd(func):
     def rpc_test_cmd(*args, **kwargs):
         try: