vhost: use external event API in get_vhost_controllers RPC call

Added spdk_vhost_call_external_event_foreach.

Continuation of patch I689226c [1]

[1] vhost: added API to call external spdk_events on vdev reactor

Change-Id: I5a404ec9de586d197e84eeda82009a803b4fa623
Signed-off-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/373262
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

include/spdk/vhost.h

@@ -57,12 +57,6 @@ struct spdk_vhost_dev;
 
 typedef int (*spdk_vhost_event_fn)(struct spdk_vhost_dev *, void *);
 
-/**
- * Get handle to next controller.
- * \param prev Previous controller or NULL to get first one.
- * \return handle to next controller ot NULL if prev was the last one.
- */
-struct spdk_vhost_dev *spdk_vhost_dev_next(struct spdk_vhost_dev *prev);
 struct spdk_vhost_dev *spdk_vhost_dev_find(const char *ctrlr_name);
 const char *spdk_vhost_dev_get_name(struct spdk_vhost_dev *ctrl);
 uint64_t spdk_vhost_dev_get_cpumask(struct spdk_vhost_dev *ctrl);
@@ -99,4 +93,22 @@ bool spdk_vhost_blk_get_readonly(struct spdk_vhost_dev *vdev);
  */
 void spdk_vhost_call_external_event(const char *ctrlr_name, spdk_vhost_event_fn fn, void *arg);
 
+/**
+ * Call function for each available vhost controller on
+ * it's reactor. This will call given function in a chain,
+ * meaning that each callback will be called after the
+ * previous one has finished. After given function has
+ * been called for all controllers, it will be called
+ * once again with first param - vhost controller - set
+ * to NULL.
+ *
+ * This function is thread safe.
+ *
+ * \param fn function to be called for each controller.
+ * The first param will be either vdev pointer or NULL.
+ * The second param is user provided argument *arg*.
+ * \param arg parameter to be passed to *fn*.
+ */
+void spdk_vhost_call_external_event_foreach(spdk_vhost_event_fn fn, void *arg);
+
 #endif /* SPDK_VHOST_H */

lib/vhost/vhost.c

@@ -482,30 +482,18 @@ spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev)
 	return 0;
 }
 
-struct spdk_vhost_dev *
+static int
-spdk_vhost_dev_next(struct spdk_vhost_dev *prev)
+spdk_vhost_dev_next(int i)
 {
-	int i = 0;
+	for (i++; i < MAX_VHOST_DEVICES; i++) {
-
-	if (prev != NULL) {
-		for (; i < MAX_VHOST_DEVICES; i++) {
-			if (g_spdk_vhost_devices[i] == prev) {
-				break;
-			}
-		}
-
-		i++;
-	}
-
-	for (; i < MAX_VHOST_DEVICES; i++) {
 		if (g_spdk_vhost_devices[i] == NULL) {
 			continue;
 		}
 
-		return g_spdk_vhost_devices[i];
+		return i;
 	}
 
-	return NULL;
+	return -1;
 }
 
 const char *
@@ -594,6 +582,34 @@ spdk_vhost_event_async_fn(void *arg1, void *arg2)
 	free(ctx);
 }
 
+static void spdk_vhost_external_event_foreach_continue(int vdev_id,
+		spdk_vhost_event_fn fn, void *arg);
+
+static void
+spdk_vhost_event_async_foreach_fn(void *arg1, void *arg2)
+{
+	struct spdk_vhost_dev_event_ctx *ctx = arg1;
+	struct spdk_vhost_dev *vdev;
+	struct spdk_event *ev;
+
+	if (pthread_mutex_trylock(&g_spdk_vhost_mutex) != 0) {
+		ev = spdk_event_allocate(spdk_env_get_current_core(),
+					 spdk_vhost_event_async_foreach_fn, arg1, arg2);
+		spdk_event_call(ev);
+		return;
+	}
+
+	vdev = g_spdk_vhost_devices[ctx->vdev_id];
+	if (vdev == ctx->vdev) {
+		ctx->cb_fn(vdev, arg2);
+	}
+
+	spdk_vhost_external_event_foreach_continue(ctx->vdev_id, ctx->cb_fn, arg2);
+	pthread_mutex_unlock(&g_spdk_vhost_mutex);
+
+	free(ctx);
+}
+
 static int
 spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn,
 		      unsigned timeout_sec, const char *errmsg)
@@ -630,10 +646,12 @@ spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn,
 }
 
 static int
-spdk_vhost_event_async_send(unsigned vdev_id, spdk_vhost_event_fn cb_fn, void *arg)
+spdk_vhost_event_async_send(unsigned vdev_id, spdk_vhost_event_fn cb_fn, void *arg,
+			    bool foreach)
 {
 	struct spdk_vhost_dev_event_ctx *ev_ctx;
 	struct spdk_event *ev;
+	spdk_event_fn fn;
 
 	ev_ctx = calloc(1, sizeof(*ev_ctx));
 	if (ev_ctx == NULL) {
@@ -645,7 +663,8 @@ spdk_vhost_event_async_send(unsigned vdev_id, spdk_vhost_event_fn cb_fn, void *a
 	ev_ctx->vdev_id = vdev_id;
 	ev_ctx->cb_fn = cb_fn;
 
-	ev = spdk_event_allocate(ev_ctx->vdev->lcore, spdk_vhost_event_async_fn, ev_ctx, arg);
+	fn = foreach ? spdk_vhost_event_async_foreach_fn : spdk_vhost_event_async_fn;
+	ev = spdk_event_allocate(ev_ctx->vdev->lcore, fn, ev_ctx, arg);
 	assert(ev);
 	spdk_event_call(ev);
 
@@ -919,12 +938,45 @@ spdk_vhost_call_external_event(const char *ctrlr_name, spdk_vhost_event_fn fn, v
 	if (vdev->lcore == -1) {
 		fn(vdev, arg);
 	} else {
-		spdk_vhost_event_async_send(vdev_id, fn, arg);
+		spdk_vhost_event_async_send(vdev_id, fn, arg, false);
 	}
 
 	pthread_mutex_unlock(&g_spdk_vhost_mutex);
 }
 
+static void
+spdk_vhost_external_event_foreach_continue(int vdev_id, spdk_vhost_event_fn fn, void *arg)
+{
+	struct spdk_vhost_dev *vdev;
+
+	vdev_id = spdk_vhost_dev_next(vdev_id);
+	if (vdev_id == -1) {
+		fn(NULL, arg);
+		return;
+	}
+
+	vdev = g_spdk_vhost_devices[vdev_id];
+	while (vdev->lcore == -1) {
+		fn(vdev, arg);
+		vdev_id = spdk_vhost_dev_next(vdev_id);
+		if (vdev_id == -1) {
+			fn(NULL, arg);
+			return;
+		}
+		vdev = g_spdk_vhost_devices[vdev_id];
+	}
+
+	spdk_vhost_event_async_send(vdev_id, fn, arg, true);
+}
+
+void
+spdk_vhost_call_external_event_foreach(spdk_vhost_event_fn fn, void *arg)
+{
+	pthread_mutex_lock(&g_spdk_vhost_mutex);
+	spdk_vhost_external_event_foreach_continue(-1, fn, arg);
+	pthread_mutex_unlock(&g_spdk_vhost_mutex);
+}
+
 void
 spdk_vhost_lock(void)
 {

lib/vhost/vhost_rpc.c

@@ -460,12 +460,49 @@ invalid:
 }
 SPDK_RPC_REGISTER("remove_vhost_controller", spdk_rpc_remove_vhost_controller)
 
+struct rpc_get_vhost_ctrlrs {
+	struct spdk_json_write_ctx *w;
+	struct spdk_jsonrpc_request *request;
+};
+
+static int
+spdk_rpc_get_vhost_controllers_cb(struct spdk_vhost_dev *vdev, void *arg)
+{
+	struct rpc_get_vhost_ctrlrs *ctx = arg;
+
+	if (vdev == NULL) {
+		spdk_json_write_array_end(ctx->w);
+		spdk_jsonrpc_end_result(ctx->request, ctx->w);
+		free(ctx);
+		return 0;
+	}
+
+	spdk_json_write_object_begin(ctx->w);
+
+	spdk_json_write_name(ctx->w, "ctrlr");
+	spdk_json_write_string(ctx->w, spdk_vhost_dev_get_name(vdev));
+
+	spdk_json_write_name(ctx->w, "cpumask");
+	spdk_json_write_string_fmt(ctx->w, "%#" PRIx64, spdk_vhost_dev_get_cpumask(vdev));
+
+	spdk_json_write_name(ctx->w, "backend_specific");
+
+	spdk_json_write_object_begin(ctx->w);
+	spdk_vhost_dump_config_json(vdev, ctx->w);
+	spdk_json_write_object_end(ctx->w);
+
+	spdk_json_write_object_end(ctx->w); // ctrl
+
+	return 0;
+}
+
+
 static void
 spdk_rpc_get_vhost_controllers(struct spdk_jsonrpc_request *request,
 			       const struct spdk_json_val *params)
 {
+	struct rpc_get_vhost_ctrlrs *ctx;
 	struct spdk_json_write_ctx *w;
-	struct spdk_vhost_dev *vdev = NULL;
 
 	if (params != NULL) {
 		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
@@ -479,26 +516,16 @@ spdk_rpc_get_vhost_controllers(struct spdk_jsonrpc_request *request,
 	}
 
 	spdk_json_write_array_begin(w);
-	while ((vdev = spdk_vhost_dev_next(vdev)) != NULL) {
-		spdk_json_write_object_begin(w);
 
-		spdk_json_write_name(w, "ctrlr");
+	ctx = calloc(1, sizeof(*ctx));
-		spdk_json_write_string(w, spdk_vhost_dev_get_name(vdev));
+	if (ctx == NULL) {
-
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, strerror(-ENOMEM));
-		spdk_json_write_name(w, "cpumask");
+		return;
-		spdk_json_write_string_fmt(w, "%#" PRIx64, spdk_vhost_dev_get_cpumask(vdev));
-
-		spdk_json_write_name(w, "backend_specific");
-
-		spdk_json_write_object_begin(w);
-		spdk_vhost_dump_config_json(vdev, w);
-		spdk_json_write_object_end(w);
-
-		spdk_json_write_object_end(w);
 	}
 
-	spdk_json_write_array_end(w);
+	ctx->w = w;
-	spdk_jsonrpc_end_result(request, w);
+	ctx->request = request;
+	spdk_vhost_call_external_event_foreach(spdk_rpc_get_vhost_controllers_cb, ctx);
 }
 SPDK_RPC_REGISTER("get_vhost_controllers", spdk_rpc_get_vhost_controllers)
 

test/unit/lib/vhost/vhost.c/vhost_ut.c

@@ -36,6 +36,7 @@
 #include "CUnit/Basic.h"
 #include "spdk_cunit.h"
 #include "spdk_internal/mock.h"
+#include "lib/test_env.c"
 
 #include "vhost.c"
 
@@ -44,7 +45,6 @@ DEFINE_STUB(spdk_event_allocate, struct spdk_event *,
 	    (uint32_t lcore, spdk_event_fn fn, void *arg1, void *arg2), NULL);
 DEFINE_STUB(spdk_mem_register, int, (void *vaddr, size_t len), 0);
 DEFINE_STUB(spdk_mem_unregister, int, (void *vaddr, size_t len), 0);
-DEFINE_STUB(spdk_vtophys, uint64_t, (void *vaddr), 1);
 DEFINE_STUB(spdk_app_get_core_mask, uint64_t, (void), 0);
 DEFINE_STUB_V(spdk_app_stop, (int rc));
 DEFINE_STUB_V(spdk_event_call, (struct spdk_event *event));