poller: add busy wait mechanism for intr

For pollers that don't natively support interrupts, using
a busy wait mechanism temporarily.
An interrupt falicity for busy wait will
be registered for non-periodic poller.
Internally, an eventfd is created to each busy wait
poller. Write the eventfd when set interrupt mode,
and only read the eventfd when set back to poll mode,
then the busy wait poller will be called repeatly
in interrupt mode.

Change-Id: Iaeae14d1ff69fd9ef7d606a0b0a70193764513e9
Signed-off-by: Liu Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6711
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot

lib/thread/thread.c

@@ -1002,8 +1002,6 @@ interrupt_timerfd_process(void *arg)
 	return poller->fn(poller->arg);
 }
 
-static void period_poller_interrupt_fini(struct spdk_poller *poller);
-
 static int
 period_poller_interrupt_init(struct spdk_poller *poller)
 {
@@ -1084,15 +1082,57 @@ period_poller_set_interrupt_mode(struct spdk_poller *poller, void *cb_arg, bool
 }
 
 static void
-period_poller_interrupt_fini(struct spdk_poller *poller)
+poller_interrupt_fini(struct spdk_poller *poller)
 {
-	SPDK_DEBUGLOG(thread, "timerfd fini for poller %s\n", poller->name);
+	SPDK_DEBUGLOG(thread, "interrupt fini for poller %s\n", poller->name);
 	assert(poller->interruptfd >= 0);
 	spdk_fd_group_remove(poller->thread->fgrp, poller->interruptfd);
 	close(poller->interruptfd);
 	poller->interruptfd = -1;
 }
 
+static int
+busy_poller_interrupt_init(struct spdk_poller *poller)
+{
+	int busy_efd;
+	int rc;
+
+	SPDK_DEBUGLOG(thread, "busy_efd init for busy poller %s\n", poller->name);
+	busy_efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+	if (busy_efd < 0) {
+		SPDK_ERRLOG("Failed to create eventfd for Poller(%s).\n", poller->name);
+		return -errno;
+	}
+
+	rc = spdk_fd_group_add(poller->thread->fgrp, busy_efd, poller->fn, poller->arg);
+	if (rc < 0) {
+		close(busy_efd);
+		return rc;
+	}
+
+	poller->interruptfd = busy_efd;
+	return 0;
+}
+
+static void
+busy_poller_set_interrupt_mode(struct spdk_poller *poller, void *cb_arg, bool interrupt_mode)
+{
+	int busy_efd = poller->interruptfd;
+	uint64_t notify = 1;
+
+	assert(busy_efd >= 0);
+
+	if (interrupt_mode) {
+		/* Write without read on eventfd will get it repeatedly triggered. */
+		if (write(busy_efd, &notify, sizeof(notify)) < 0) {
+			SPDK_ERRLOG("Failed to set busy wait for Poller(%s).\n", poller->name);
+		}
+	} else {
+		/* Read on eventfd will clear its level triggering. */
+		read(busy_efd, &notify, sizeof(notify));
+	}
+}
+
 #else
 
 static int
@@ -1107,9 +1147,21 @@ period_poller_set_interrupt_mode(struct spdk_poller *poller, void *cb_arg, bool
 }
 
 static void
-period_poller_interrupt_fini(struct spdk_poller *poller)
+poller_interrupt_fini(struct spdk_poller *poller)
 {
 }
+
+static int
+busy_poller_interrupt_init(struct spdk_poller *poller)
+{
+	return -ENOTSUP;
+}
+
+static void
+busy_poller_set_interrupt_mode(struct spdk_poller *poller, void *cb_arg, bool interrupt_mode)
+{
+}
+
 #endif
 
 void
@@ -1125,6 +1177,16 @@ spdk_poller_register_interrupt(struct spdk_poller *poller,
 		return;
 	}
 
+	/* when a poller is created we don't know if the user is ever going to
+	 * enable interrupts on it by calling this function, so the poller
+	 * registration function has to immediately create a interruptfd.
+	 * When this function does get called by user, we have to then destroy
+	 * that interruptfd.
+	 */
+	if (poller->set_intr_cb_fn && poller->interruptfd >= 0) {
+		poller_interrupt_fini(poller);
+	}
+
 	poller->set_intr_cb_fn = cb_fn;
 	poller->set_intr_cb_arg = cb_arg;
 
@@ -1183,17 +1245,31 @@ poller_register(spdk_poller_fn fn,
 		poller->period_ticks = 0;
 	}
 
-	if (period_microseconds && spdk_interrupt_mode_is_enabled()) {
+	if (spdk_interrupt_mode_is_enabled()) {
 		int rc;
 
-		rc = period_poller_interrupt_init(poller);
+		if (period_microseconds) {
-		if (rc < 0) {
+			rc = period_poller_interrupt_init(poller);
-			SPDK_ERRLOG("Failed to register timerfd for periodic poller: %s\n", spdk_strerror(-rc));
+			if (rc < 0) {
-			free(poller);
+				SPDK_ERRLOG("Failed to register interruptfd for periodic poller: %s\n", spdk_strerror(-rc));
-			return NULL;
+				free(poller);
-		}
+				return NULL;
+			}
 
-		spdk_poller_register_interrupt(poller, period_poller_set_interrupt_mode, NULL);
+			spdk_poller_register_interrupt(poller, period_poller_set_interrupt_mode, NULL);
+		} else {
+			/* If the poller doesn't have a period, create interruptfd that's always
+			 * busy automatically when runnning in interrupt mode.
+			 */
+			rc = busy_poller_interrupt_init(poller);
+			if (rc > 0) {
+				SPDK_ERRLOG("Failed to register interruptfd for busy poller: %s\n", spdk_strerror(-rc));
+				free(poller);
+				return NULL;
+			}
+
+			spdk_poller_register_interrupt(poller, busy_poller_set_interrupt_mode, NULL);
+		}
 	}
 
 	thread_insert_poller(thread, poller);
@@ -1244,7 +1320,7 @@ spdk_poller_unregister(struct spdk_poller **ppoller)
 	}
 
 	if (spdk_interrupt_mode_is_enabled() && poller->interruptfd >= 0) {
-		period_poller_interrupt_fini(poller);
+		poller_interrupt_fini(poller);
 	}
 
 	/* If the poller was paused, put it on the active_pollers list so that