From 8323be7df86f30a0bbeb25c8bdd440dddb30ee07 Mon Sep 17 00:00:00 2001 From: Dariusz Stojaczyk Date: Tue, 8 Aug 2017 18:52:53 +0200 Subject: [PATCH] vhost: use external event API in get_vhost_controllers RPC call Added spdk_vhost_call_external_event_foreach. Continuation of patch I689226c [1] [1] vhost: added API to call external spdk_events on vdev reactor Change-Id: I5a404ec9de586d197e84eeda82009a803b4fa623 Signed-off-by: Dariusz Stojaczyk Reviewed-on: https://review.gerrithub.io/373262 Tested-by: SPDK Automated Test System Reviewed-by: Pawel Wodkowski Reviewed-by: Jim Harris --- include/spdk/vhost.h | 24 +++++-- lib/vhost/vhost.c | 92 ++++++++++++++++++++------ lib/vhost/vhost_rpc.c | 63 +++++++++++++----- test/unit/lib/vhost/vhost.c/vhost_ut.c | 2 +- 4 files changed, 136 insertions(+), 45 deletions(-) diff --git a/include/spdk/vhost.h b/include/spdk/vhost.h index 69074386e..4342e907d 100644 --- a/include/spdk/vhost.h +++ b/include/spdk/vhost.h @@ -57,12 +57,6 @@ struct spdk_vhost_dev; typedef int (*spdk_vhost_event_fn)(struct spdk_vhost_dev *, void *); -/** - * Get handle to next controller. - * \param prev Previous controller or NULL to get first one. - * \return handle to next controller ot NULL if prev was the last one. - */ -struct spdk_vhost_dev *spdk_vhost_dev_next(struct spdk_vhost_dev *prev); struct spdk_vhost_dev *spdk_vhost_dev_find(const char *ctrlr_name); const char *spdk_vhost_dev_get_name(struct spdk_vhost_dev *ctrl); uint64_t spdk_vhost_dev_get_cpumask(struct spdk_vhost_dev *ctrl); @@ -99,4 +93,22 @@ bool spdk_vhost_blk_get_readonly(struct spdk_vhost_dev *vdev); */ void spdk_vhost_call_external_event(const char *ctrlr_name, spdk_vhost_event_fn fn, void *arg); +/** + * Call function for each available vhost controller on + * it's reactor. This will call given function in a chain, + * meaning that each callback will be called after the + * previous one has finished. After given function has + * been called for all controllers, it will be called + * once again with first param - vhost controller - set + * to NULL. + * + * This function is thread safe. + * + * \param fn function to be called for each controller. + * The first param will be either vdev pointer or NULL. + * The second param is user provided argument *arg*. + * \param arg parameter to be passed to *fn*. + */ +void spdk_vhost_call_external_event_foreach(spdk_vhost_event_fn fn, void *arg); + #endif /* SPDK_VHOST_H */ diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index 28c76cb32..deebe94e9 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -482,30 +482,18 @@ spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev) return 0; } -struct spdk_vhost_dev * -spdk_vhost_dev_next(struct spdk_vhost_dev *prev) +static int +spdk_vhost_dev_next(int i) { - int i = 0; - - if (prev != NULL) { - for (; i < MAX_VHOST_DEVICES; i++) { - if (g_spdk_vhost_devices[i] == prev) { - break; - } - } - - i++; - } - - for (; i < MAX_VHOST_DEVICES; i++) { + for (i++; i < MAX_VHOST_DEVICES; i++) { if (g_spdk_vhost_devices[i] == NULL) { continue; } - return g_spdk_vhost_devices[i]; + return i; } - return NULL; + return -1; } const char * @@ -594,6 +582,34 @@ spdk_vhost_event_async_fn(void *arg1, void *arg2) free(ctx); } +static void spdk_vhost_external_event_foreach_continue(int vdev_id, + spdk_vhost_event_fn fn, void *arg); + +static void +spdk_vhost_event_async_foreach_fn(void *arg1, void *arg2) +{ + struct spdk_vhost_dev_event_ctx *ctx = arg1; + struct spdk_vhost_dev *vdev; + struct spdk_event *ev; + + if (pthread_mutex_trylock(&g_spdk_vhost_mutex) != 0) { + ev = spdk_event_allocate(spdk_env_get_current_core(), + spdk_vhost_event_async_foreach_fn, arg1, arg2); + spdk_event_call(ev); + return; + } + + vdev = g_spdk_vhost_devices[ctx->vdev_id]; + if (vdev == ctx->vdev) { + ctx->cb_fn(vdev, arg2); + } + + spdk_vhost_external_event_foreach_continue(ctx->vdev_id, ctx->cb_fn, arg2); + pthread_mutex_unlock(&g_spdk_vhost_mutex); + + free(ctx); +} + static int spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn, unsigned timeout_sec, const char *errmsg) @@ -630,10 +646,12 @@ spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn, } static int -spdk_vhost_event_async_send(unsigned vdev_id, spdk_vhost_event_fn cb_fn, void *arg) +spdk_vhost_event_async_send(unsigned vdev_id, spdk_vhost_event_fn cb_fn, void *arg, + bool foreach) { struct spdk_vhost_dev_event_ctx *ev_ctx; struct spdk_event *ev; + spdk_event_fn fn; ev_ctx = calloc(1, sizeof(*ev_ctx)); if (ev_ctx == NULL) { @@ -645,7 +663,8 @@ spdk_vhost_event_async_send(unsigned vdev_id, spdk_vhost_event_fn cb_fn, void *a ev_ctx->vdev_id = vdev_id; ev_ctx->cb_fn = cb_fn; - ev = spdk_event_allocate(ev_ctx->vdev->lcore, spdk_vhost_event_async_fn, ev_ctx, arg); + fn = foreach ? spdk_vhost_event_async_foreach_fn : spdk_vhost_event_async_fn; + ev = spdk_event_allocate(ev_ctx->vdev->lcore, fn, ev_ctx, arg); assert(ev); spdk_event_call(ev); @@ -919,12 +938,45 @@ spdk_vhost_call_external_event(const char *ctrlr_name, spdk_vhost_event_fn fn, v if (vdev->lcore == -1) { fn(vdev, arg); } else { - spdk_vhost_event_async_send(vdev_id, fn, arg); + spdk_vhost_event_async_send(vdev_id, fn, arg, false); } pthread_mutex_unlock(&g_spdk_vhost_mutex); } +static void +spdk_vhost_external_event_foreach_continue(int vdev_id, spdk_vhost_event_fn fn, void *arg) +{ + struct spdk_vhost_dev *vdev; + + vdev_id = spdk_vhost_dev_next(vdev_id); + if (vdev_id == -1) { + fn(NULL, arg); + return; + } + + vdev = g_spdk_vhost_devices[vdev_id]; + while (vdev->lcore == -1) { + fn(vdev, arg); + vdev_id = spdk_vhost_dev_next(vdev_id); + if (vdev_id == -1) { + fn(NULL, arg); + return; + } + vdev = g_spdk_vhost_devices[vdev_id]; + } + + spdk_vhost_event_async_send(vdev_id, fn, arg, true); +} + +void +spdk_vhost_call_external_event_foreach(spdk_vhost_event_fn fn, void *arg) +{ + pthread_mutex_lock(&g_spdk_vhost_mutex); + spdk_vhost_external_event_foreach_continue(-1, fn, arg); + pthread_mutex_unlock(&g_spdk_vhost_mutex); +} + void spdk_vhost_lock(void) { diff --git a/lib/vhost/vhost_rpc.c b/lib/vhost/vhost_rpc.c index 84c912f8a..f2fcb2b8f 100644 --- a/lib/vhost/vhost_rpc.c +++ b/lib/vhost/vhost_rpc.c @@ -460,12 +460,49 @@ invalid: } SPDK_RPC_REGISTER("remove_vhost_controller", spdk_rpc_remove_vhost_controller) +struct rpc_get_vhost_ctrlrs { + struct spdk_json_write_ctx *w; + struct spdk_jsonrpc_request *request; +}; + +static int +spdk_rpc_get_vhost_controllers_cb(struct spdk_vhost_dev *vdev, void *arg) +{ + struct rpc_get_vhost_ctrlrs *ctx = arg; + + if (vdev == NULL) { + spdk_json_write_array_end(ctx->w); + spdk_jsonrpc_end_result(ctx->request, ctx->w); + free(ctx); + return 0; + } + + spdk_json_write_object_begin(ctx->w); + + spdk_json_write_name(ctx->w, "ctrlr"); + spdk_json_write_string(ctx->w, spdk_vhost_dev_get_name(vdev)); + + spdk_json_write_name(ctx->w, "cpumask"); + spdk_json_write_string_fmt(ctx->w, "%#" PRIx64, spdk_vhost_dev_get_cpumask(vdev)); + + spdk_json_write_name(ctx->w, "backend_specific"); + + spdk_json_write_object_begin(ctx->w); + spdk_vhost_dump_config_json(vdev, ctx->w); + spdk_json_write_object_end(ctx->w); + + spdk_json_write_object_end(ctx->w); // ctrl + + return 0; +} + + static void spdk_rpc_get_vhost_controllers(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { + struct rpc_get_vhost_ctrlrs *ctx; struct spdk_json_write_ctx *w; - struct spdk_vhost_dev *vdev = NULL; if (params != NULL) { spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, @@ -479,26 +516,16 @@ spdk_rpc_get_vhost_controllers(struct spdk_jsonrpc_request *request, } spdk_json_write_array_begin(w); - while ((vdev = spdk_vhost_dev_next(vdev)) != NULL) { - spdk_json_write_object_begin(w); - spdk_json_write_name(w, "ctrlr"); - spdk_json_write_string(w, spdk_vhost_dev_get_name(vdev)); - - spdk_json_write_name(w, "cpumask"); - spdk_json_write_string_fmt(w, "%#" PRIx64, spdk_vhost_dev_get_cpumask(vdev)); - - spdk_json_write_name(w, "backend_specific"); - - spdk_json_write_object_begin(w); - spdk_vhost_dump_config_json(vdev, w); - spdk_json_write_object_end(w); - - spdk_json_write_object_end(w); + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, strerror(-ENOMEM)); + return; } - spdk_json_write_array_end(w); - spdk_jsonrpc_end_result(request, w); + ctx->w = w; + ctx->request = request; + spdk_vhost_call_external_event_foreach(spdk_rpc_get_vhost_controllers_cb, ctx); } SPDK_RPC_REGISTER("get_vhost_controllers", spdk_rpc_get_vhost_controllers) diff --git a/test/unit/lib/vhost/vhost.c/vhost_ut.c b/test/unit/lib/vhost/vhost.c/vhost_ut.c index b7162f43a..1d55e2b67 100644 --- a/test/unit/lib/vhost/vhost.c/vhost_ut.c +++ b/test/unit/lib/vhost/vhost.c/vhost_ut.c @@ -36,6 +36,7 @@ #include "CUnit/Basic.h" #include "spdk_cunit.h" #include "spdk_internal/mock.h" +#include "lib/test_env.c" #include "vhost.c" @@ -44,7 +45,6 @@ DEFINE_STUB(spdk_event_allocate, struct spdk_event *, (uint32_t lcore, spdk_event_fn fn, void *arg1, void *arg2), NULL); DEFINE_STUB(spdk_mem_register, int, (void *vaddr, size_t len), 0); DEFINE_STUB(spdk_mem_unregister, int, (void *vaddr, size_t len), 0); -DEFINE_STUB(spdk_vtophys, uint64_t, (void *vaddr), 1); DEFINE_STUB(spdk_app_get_core_mask, uint64_t, (void), 0); DEFINE_STUB_V(spdk_app_stop, (int rc)); DEFINE_STUB_V(spdk_event_call, (struct spdk_event *event));