vhost: generalized vhost events

Rework reactor<->pthread comunication logic, so we can reuse it in other
backends.

Change-Id: Id0f02fc9e1e0d9540f8efebdb515384b5714c966
Signed-off-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-on: https://review.gerrithub.io/362548
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Piotr Pelpliński <piotr.pelplinski@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/vhost/vhost.c

@@ -35,6 +35,7 @@
 
 #include "spdk/env.h"
 #include "spdk/likely.h"
+#include "spdk/util.h"
 
 #include "spdk/vhost.h"
 #include "vhost_internal.h"
@@ -621,4 +622,67 @@ spdk_vhost_shutdown_cb(void)
 	pthread_detach(tid);
 }
 
+static void
+vhost_timed_event_fn(void *arg1, void *arg2)
+{
+	struct spdk_vhost_timed_event *ev = arg1;
+
+	if (ev->cb_fn) {
+		ev->cb_fn(arg2);
+	}
+
+	sem_post(&ev->sem);
+}
+
+static void
+vhost_timed_event_init(struct spdk_vhost_timed_event *ev, int32_t lcore,
+		       spdk_vhost_timed_event_fn cb_fn, void *arg, unsigned timeout_sec)
+{
+	/* No way to free spdk event so don't allow to use it again without calling, waiting. */
+	assert(ev->spdk_event == NULL);
+
+	if (sem_init(&ev->sem, 0, 0) < 0)
+		SPDK_ERRLOG("Failed to initialize semaphore for vhost timed event\n");
+
+	ev->cb_fn = cb_fn;
+	clock_gettime(CLOCK_REALTIME, &ev->timeout);
+	ev->timeout.tv_sec += timeout_sec;
+	ev->spdk_event = spdk_event_allocate(lcore, vhost_timed_event_fn, ev, arg);
+}
+
+void
+spdk_vhost_timed_event_init(struct spdk_vhost_timed_event *ev, int32_t lcore,
+			    spdk_vhost_timed_event_fn cb_fn, void *arg, unsigned timeout_sec)
+{
+	vhost_timed_event_init(ev, lcore, cb_fn, arg, timeout_sec);
+}
+
+void
+spdk_vhost_timed_event_send(int32_t lcore, spdk_vhost_timed_event_fn cb_fn, void *arg,
+			    unsigned timeout_sec, const char *errmsg)
+{
+	struct spdk_vhost_timed_event ev = {0};
+
+	vhost_timed_event_init(&ev, lcore, cb_fn, arg, timeout_sec);
+	spdk_event_call(ev.spdk_event);
+	spdk_vhost_timed_event_wait(&ev, errmsg);
+}
+
+void
+spdk_vhost_timed_event_wait(struct spdk_vhost_timed_event *ev, const char *errmsg)
+{
+	int rc;
+
+	assert(ev->spdk_event != NULL);
+
+	rc = sem_timedwait(&ev->sem, &ev->timeout);
+	if (rc != 0) {
+		SPDK_ERRLOG("Timout waiting for event: %s.\n", errmsg);
+		abort();
+	}
+
+	ev->spdk_event = NULL;
+	sem_destroy(&ev->sem);
+}
+
 SPDK_LOG_REGISTER_TRACE_FLAG("vhost_ring", SPDK_TRACE_VHOST_RING)

lib/vhost/vhost_internal.h

@@ -99,4 +99,28 @@ int spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev);
 struct spdk_vhost_dev *spdk_vhost_dev_load(int vid);
 void spdk_vhost_dev_unload(struct spdk_vhost_dev *dev);
 
+typedef void (*spdk_vhost_timed_event_fn)(void *);
+
+struct spdk_vhost_timed_event {
+	/** User callback function to be executed on given lcore. */
+	spdk_vhost_timed_event_fn cb_fn;
+
+	/** Semaphore used to signal that event is done. */
+	sem_t sem;
+
+	/** Timout specified during initialization. */
+	struct timespec timeout;
+
+	/** Event object that can be passed to *spdk_event_call()*. */
+	struct spdk_event *spdk_event;
+};
+
+void spdk_vhost_timed_event_init(struct spdk_vhost_timed_event *ev, int32_t lcore,
+				 spdk_vhost_timed_event_fn cb_fn, void *arg, unsigned timeout_sec);
+
+void spdk_vhost_timed_event_send(int32_t lcore, spdk_vhost_timed_event_fn cn_fn, void *arg,
+				 unsigned timeout_sec, const char *errmsg);
+void spdk_vhost_timed_event_wait(struct spdk_vhost_timed_event *event, const char *errmsg);
+
+
 #endif /* SPDK_VHOST_INTERNAL_H */

lib/vhost/vhost_scsi.c

@@ -535,39 +535,9 @@ vdev_worker(void *arg)
 }
 
 static void
-vdev_event_done_cb(void *arg1, void *arg2)
+add_vdev_cb(void *arg)
 {
-	sem_post((sem_t *)arg2);
+	struct spdk_vhost_scsi_dev *svdev = arg;
-}
-
-static struct spdk_event *
-vhost_sem_event_alloc(uint32_t core, spdk_event_fn fn, void *arg1, sem_t *sem)
-{
-	if (sem_init(sem, 0, 0) < 0)
-		rte_panic("Failed to initialize semaphore.");
-
-	return spdk_event_allocate(core, fn, arg1, sem);
-}
-
-static int
-vhost_sem_timedwait(sem_t *sem, unsigned sec)
-{
-	struct timespec timeout;
-	int rc;
-
-	clock_gettime(CLOCK_REALTIME, &timeout);
-	timeout.tv_sec += sec;
-
-	rc = sem_timedwait(sem, &timeout);
-	sem_destroy(sem);
-
-	return rc;
-}
-
-static void
-add_vdev_cb(void *arg1, void *arg2)
-{
-	struct spdk_vhost_scsi_dev *svdev = arg1;
 	struct spdk_vhost_dev *vdev = &svdev->vdev;
 	uint32_t i;
 
@@ -584,13 +554,12 @@ add_vdev_cb(void *arg1, void *arg2)
 	spdk_poller_register(&svdev->requestq_poller, vdev_worker, svdev, vdev->lcore, 0);
 	spdk_poller_register(&svdev->controlq_poller, vdev_controlq_worker, svdev, vdev->lcore,
 			     CONTROLQ_POLL_PERIOD_US);
-	sem_post((sem_t *)arg2);
 }
 
 static void
-remove_vdev_cb(void *arg1, void *arg2)
+remove_vdev_cb(void *arg)
 {
-	struct spdk_vhost_scsi_dev *svdev = arg1;
+	struct spdk_vhost_scsi_dev *svdev = arg;
 	uint32_t i;
 
 	for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
@@ -602,8 +571,6 @@ remove_vdev_cb(void *arg1, void *arg2)
 
 	SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", svdev->vdev.name);
 	spdk_vhost_dev_mem_unregister(&svdev->vdev);
-
-	sem_post((sem_t *)arg2);
 }
 
 int
@@ -821,18 +788,13 @@ static int
 new_device(int vid)
 {
 	struct spdk_vhost_dev *vdev = NULL;
-	struct spdk_event *event;
-	sem_t added;
 
 	vdev = spdk_vhost_dev_load(vid);
 	if (vdev == NULL) {
 		return -1;
 	}
 
-	event = vhost_sem_event_alloc(vdev->lcore, add_vdev_cb, vdev, &added);
+	spdk_vhost_timed_event_send(vdev->lcore, add_vdev_cb, vdev, 1, "add scsi vdev");
-	spdk_event_call(event);
-	if (vhost_sem_timedwait(&added, 1))
-		rte_panic("Failed to register new device '%s'\n", vdev->name);
 	return 0;
 }
 
@@ -841,8 +803,7 @@ destroy_device(int vid)
 {
 	struct spdk_vhost_scsi_dev *svdev;
 	struct spdk_vhost_dev *vdev;
-	struct spdk_event *event;
+	struct spdk_vhost_timed_event event = {0};
-	sem_t done_sem;
 	uint32_t i;
 
 	vdev = spdk_vhost_dev_find_by_vid(vid);
@@ -851,15 +812,13 @@ destroy_device(int vid)
 	}
 	svdev = (struct spdk_vhost_scsi_dev *) vdev;
 
-	event = vhost_sem_event_alloc(vdev->lcore, vdev_event_done_cb, NULL, &done_sem);
+	spdk_vhost_timed_event_init(&event, vdev->lcore, NULL, NULL, 1);
-	spdk_poller_unregister(&svdev->requestq_poller, event);
+	spdk_poller_unregister(&svdev->requestq_poller, event.spdk_event);
-	if (vhost_sem_timedwait(&done_sem, 1))
+	spdk_vhost_timed_event_wait(&event, "unregister request queue poller");
-		rte_panic("%s: failed to unregister request queue poller.\n", vdev->name);
 
-	event = vhost_sem_event_alloc(vdev->lcore, vdev_event_done_cb, NULL, &done_sem);
+	spdk_vhost_timed_event_init(&event, vdev->lcore, NULL, NULL, 1);
-	spdk_poller_unregister(&svdev->controlq_poller, event);
+	spdk_poller_unregister(&svdev->controlq_poller, event.spdk_event);
-	if (vhost_sem_timedwait(&done_sem, 1))
+	spdk_vhost_timed_event_wait(&event, "unregister controll queue poller");
-		rte_panic("%s: failed to unregister control queue poller.\n", vdev->name);
 
 	/* Wait for all tasks to finish */
 	for (i = 1000; i && vdev->task_cnt > 0; i--) {
@@ -870,10 +829,8 @@ destroy_device(int vid)
 		rte_panic("%s: pending tasks did not finish in 1s.\n", vdev->name);
 	}
 
-	event = vhost_sem_event_alloc(vdev->lcore, remove_vdev_cb, svdev, &done_sem);
+
-	spdk_event_call(event);
+	spdk_vhost_timed_event_send(vdev->lcore, remove_vdev_cb, svdev, 1, "remove scsi vdev");
-	if (vhost_sem_timedwait(&done_sem, 1))
-		rte_panic("%s: failed to unregister poller.\n", vdev->name);
 
 	spdk_vhost_dev_unload(vdev);
 }