From c7cf48ddbe17959ae961d480f507c164ecab32ca Mon Sep 17 00:00:00 2001 From: Liu Xiaodong Date: Wed, 30 Dec 2020 06:48:29 -0500 Subject: [PATCH] poller: add register interrupt function Defined callback for spdk_poller to adapt itself to set interrupt or poll mode. The callback can be registered to spdk_poller by new function `spdk_poller_register_interrupt` Interrupt callback operations for period poller are implemented, so period pollers now are interruptable. Change-Id: I2aa6ebfdd75f76b85a70af7e42530be4131ddc8a Signed-off-by: Liu Xiaodong Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5752 Reviewed-by: Konrad Sztyber Reviewed-by: Ben Walker Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot --- CHANGELOG.md | 4 + include/spdk/thread.h | 24 ++++++ include/spdk_internal/thread.h | 2 + lib/thread/spdk_thread.map | 1 + lib/thread/thread.c | 147 +++++++++++++++++++++++++-------- 5 files changed, 143 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99136b341..6b7468d12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,10 @@ Added `spdk_thread_set_interrupt_mode` function in order to set present spdk_the interrupt mode or back to poll mode. It is valid only when thread interrupt facility is enabled by spdk_interrupt_mode_enable(). +Added `spdk_poller_register_interrupt` function to mark that the poller is capable of +entering interrupt mode. Callback function will be called when the poller must transition +into or out of interrupt mode. + ### iscsi A security vulnerability has been identified and fixed in the SPDK iSCSI target. diff --git a/include/spdk/thread.h b/include/spdk/thread.h index 57153afb3..3f0459670 100644 --- a/include/spdk/thread.h +++ b/include/spdk/thread.h @@ -148,6 +148,30 @@ typedef struct spdk_poller *(*spdk_start_poller)(void *thread_ctx, */ typedef void (*spdk_stop_poller)(struct spdk_poller *poller, void *thread_ctx); +/** + * Callback function to set poller into interrupt mode or back to poll mode. + * + * \param poller Poller to set interrupt or poll mode. + * \param cb_arg Argument passed to the callback function. + * \param interrupt_mode Set interrupt mode for true, or poll mode for false + */ +typedef void (*spdk_poller_set_interrupt_mode_cb)(struct spdk_poller *poller, void *cb_arg, + bool interrupt_mode); + +/** + * Mark that the poller is capable of entering interrupt mode. + * + * When registering the poller set interrupt callback, the callback will get + * executed immediately if its spdk_thread is in the interrupt mode. + * + * \param poller The poller to register callback function. + * \param cb_fn Callback function called when the poller must transition into or out of interrupt mode + * \param cb_arg Argument passed to the callback function. + */ +void spdk_poller_register_interrupt(struct spdk_poller *poller, + spdk_poller_set_interrupt_mode_cb cb_fn, + void *cb_arg); + /** * I/O channel creation callback. * diff --git a/include/spdk_internal/thread.h b/include/spdk_internal/thread.h index 5822fd6b4..875282d50 100644 --- a/include/spdk_internal/thread.h +++ b/include/spdk_internal/thread.h @@ -75,6 +75,8 @@ struct spdk_poller { void *arg; struct spdk_thread *thread; int timerfd; + spdk_poller_set_interrupt_mode_cb set_intr_cb_fn; + void *set_intr_cb_arg; char name[SPDK_MAX_POLLER_NAME_LEN + 1]; }; diff --git a/lib/thread/spdk_thread.map b/lib/thread/spdk_thread.map index feedd551b..4f16fa7b8 100644 --- a/lib/thread/spdk_thread.map +++ b/lib/thread/spdk_thread.map @@ -35,6 +35,7 @@ spdk_poller_unregister; spdk_poller_pause; spdk_poller_resume; + spdk_poller_register_interrupt; spdk_io_device_register; spdk_io_device_unregister; spdk_get_io_channel; diff --git a/lib/thread/thread.c b/lib/thread/thread.c index d86786ac8..26f5a5b38 100644 --- a/lib/thread/thread.c +++ b/lib/thread/thread.c @@ -1010,12 +1010,6 @@ period_poller_interrupt_init(struct spdk_poller *poller) struct spdk_fd_group *fgrp = poller->thread->fgrp; int timerfd; int rc; - uint64_t now_tick = spdk_get_ticks(); - uint64_t ticks = spdk_get_ticks_hz(); - int ret; - struct itimerspec new_tv; - - assert(poller->period_ticks != 0); SPDK_DEBUGLOG(thread, "timerfd init for period poller %s\n", poller->name); timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC); @@ -1031,32 +1025,64 @@ period_poller_interrupt_init(struct spdk_poller *poller) } poller->timerfd = timerfd; - - /* Set repeated timer expirations */ - new_tv.it_interval.tv_sec = poller->period_ticks / ticks; - new_tv.it_interval.tv_nsec = poller->period_ticks % ticks * SPDK_SEC_TO_NSEC / ticks; - - /* Update next expiration */ - if (poller->next_run_tick == 0) { - poller->next_run_tick = now_tick + poller->period_ticks; - } else if (poller->next_run_tick < now_tick) { - poller->next_run_tick = now_tick; - } - - new_tv.it_value.tv_sec = (poller->next_run_tick - now_tick) / ticks; - new_tv.it_value.tv_nsec = (poller->next_run_tick - now_tick) % ticks * SPDK_SEC_TO_NSEC / ticks; - - ret = timerfd_settime(timerfd, 0, &new_tv, NULL); - if (ret < 0) { - rc = -errno; - SPDK_ERRLOG("Failed to arm timerfd: error(%d)\n", errno); - period_poller_interrupt_fini(poller); - return rc; - } - return 0; } +static void +period_poller_set_interrupt_mode(struct spdk_poller *poller, void *cb_arg, bool interrupt_mode) +{ + int timerfd = poller->timerfd; + uint64_t now_tick = spdk_get_ticks(); + uint64_t ticks = spdk_get_ticks_hz(); + int ret; + struct itimerspec new_tv = {}; + struct itimerspec old_tv = {}; + + assert(poller->period_ticks != 0); + assert(timerfd >= 0); + + SPDK_DEBUGLOG(thread, "timerfd set poller %s into %s mode\n", poller->name, + interrupt_mode ? "interrupt" : "poll"); + + if (interrupt_mode) { + /* Set repeated timer expiration */ + new_tv.it_interval.tv_sec = poller->period_ticks / ticks; + new_tv.it_interval.tv_nsec = poller->period_ticks % ticks * SPDK_SEC_TO_NSEC / ticks; + + /* Update next timer expiration */ + if (poller->next_run_tick == 0) { + poller->next_run_tick = now_tick + poller->period_ticks; + } else if (poller->next_run_tick < now_tick) { + poller->next_run_tick = now_tick; + } + + new_tv.it_value.tv_sec = (poller->next_run_tick - now_tick) / ticks; + new_tv.it_value.tv_nsec = (poller->next_run_tick - now_tick) % ticks * SPDK_SEC_TO_NSEC / ticks; + + ret = timerfd_settime(timerfd, 0, &new_tv, NULL); + if (ret < 0) { + SPDK_ERRLOG("Failed to arm timerfd: error(%d)\n", errno); + assert(false); + } + } else { + /* Disarm the timer */ + ret = timerfd_settime(timerfd, 0, &new_tv, &old_tv); + if (ret < 0) { + /* timerfd_settime's failure indicates that the timerfd is in error */ + SPDK_ERRLOG("Failed to disarm timerfd: error(%d)\n", errno); + assert(false); + } + + /* In order to reuse poller_insert_timer, fix now_tick, so next_run_tick would be + * now_tick + ticks * old_tv.it_value.tv_sec + (ticks * old_tv.it_value.tv_nsec) / SPDK_SEC_TO_NSEC + */ + now_tick = now_tick - poller->period_ticks + ticks * old_tv.it_value.tv_sec + \ + (ticks * old_tv.it_value.tv_nsec) / SPDK_SEC_TO_NSEC; + TAILQ_REMOVE(&poller->thread->timed_pollers, poller, tailq); + poller_insert_timer(poller->thread, poller, now_tick); + } +} + static void period_poller_interrupt_fini(struct spdk_poller *poller) { @@ -1075,12 +1101,39 @@ period_poller_interrupt_init(struct spdk_poller *poller) return -ENOTSUP; } +static void +period_poller_set_interrupt_mode(struct spdk_poller *poller, void *cb_arg, bool interrupt_mode) +{ +} + static void period_poller_interrupt_fini(struct spdk_poller *poller) { } #endif +void +spdk_poller_register_interrupt(struct spdk_poller *poller, + spdk_poller_set_interrupt_mode_cb cb_fn, + void *cb_arg) +{ + assert(poller != NULL); + assert(cb_fn != NULL); + assert(spdk_get_thread() == poller->thread); + + if (!spdk_interrupt_mode_is_enabled()) { + return; + } + + poller->set_intr_cb_fn = cb_fn; + poller->set_intr_cb_arg = cb_arg; + + /* Set poller into interrupt mode if thread is in interrupt. */ + if (poller->thread->in_interrupt) { + poller->set_intr_cb_fn(poller, poller->set_intr_cb_arg, true); + } +} + static struct spdk_poller * poller_register(spdk_poller_fn fn, void *arg, @@ -1139,6 +1192,8 @@ poller_register(spdk_poller_fn fn, free(poller); return NULL; } + + spdk_poller_register_interrupt(poller, period_poller_set_interrupt_mode, NULL); } thread_insert_poller(thread, poller); @@ -1365,10 +1420,27 @@ spdk_for_each_thread(spdk_msg_fn fn, void *ctx, spdk_msg_fn cpl) assert(rc == 0); } +static inline void +poller_set_interrupt_mode(struct spdk_poller *poller, bool interrupt_mode) +{ + if (poller->state == SPDK_POLLER_STATE_UNREGISTERED) { + return; + } + + if (!poller->set_intr_cb_fn) { + SPDK_ERRLOG("Poller(%s) doesn't support set interrupt mode.\n", poller->name); + assert(false); + return; + } + + poller->set_intr_cb_fn(poller, poller->set_intr_cb_arg, interrupt_mode); +} + void spdk_thread_set_interrupt_mode(bool enable_interrupt) { struct spdk_thread *thread = _get_thread(); + struct spdk_poller *poller, *tmp; assert(thread); assert(spdk_interrupt_mode_is_enabled()); @@ -1377,12 +1449,17 @@ spdk_thread_set_interrupt_mode(bool enable_interrupt) return; } - /* Currently, only spdk_thread without pollers can set interrupt mode. - * TODO: remove it after adding interrupt mode switch into poller. - */ - assert(TAILQ_EMPTY(&thread->timed_pollers)); - assert(TAILQ_EMPTY(&thread->active_pollers)); - assert(TAILQ_EMPTY(&thread->paused_pollers)); + /* Set pollers to expected mode */ + TAILQ_FOREACH_SAFE(poller, &thread->timed_pollers, tailq, tmp) { + poller_set_interrupt_mode(poller, enable_interrupt); + } + TAILQ_FOREACH_SAFE(poller, &thread->active_pollers, tailq, tmp) { + poller_set_interrupt_mode(poller, enable_interrupt); + } + /* All paused pollers will go to work in interrupt mode */ + TAILQ_FOREACH_SAFE(poller, &thread->paused_pollers, tailq, tmp) { + poller_set_interrupt_mode(poller, enable_interrupt); + } thread->in_interrupt = enable_interrupt; return;