From 316f9ea3f59ab499b627e0d4d4c4349778a6d4a0 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Mon, 12 Dec 2022 16:38:09 +0100 Subject: [PATCH] accel: pull data if srcbuf is in remote memory domain If the source buffer is from a remote memory domain, we will now pull it to the temporary bounce buffer before a task is executed. Signed-off-by: Konrad Sztyber Change-Id: I476684a4359410c69dd69a2b425b9e61d4c55a7e Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15947 Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Ben Walker --- lib/accel/accel.c | 51 ++++++++++++++++++++++++++ test/unit/lib/accel/accel.c/accel_ut.c | 6 +++ 2 files changed, 57 insertions(+) diff --git a/lib/accel/accel.c b/lib/accel/accel.c index b42ed0c54..b25d45151 100644 --- a/lib/accel/accel.c +++ b/lib/accel/accel.c @@ -67,6 +67,8 @@ enum accel_sequence_state { ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF, + ACCEL_SEQUENCE_STATE_PULL_DATA, + ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA, ACCEL_SEQUENCE_STATE_EXEC_TASK, ACCEL_SEQUENCE_STATE_AWAIT_TASK, ACCEL_SEQUENCE_STATE_COMPLETE_TASK, @@ -81,6 +83,8 @@ __attribute__((unused)) = { [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf", [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf", [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf", + [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data", + [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data", [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task", [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task", [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task", @@ -247,6 +251,8 @@ _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, voi accel_task->cb_fn = cb_fn; accel_task->cb_arg = cb_arg; accel_task->accel_ch = accel_ch; + accel_task->bounce.s.orig_iovs = NULL; + accel_task->bounce.d.orig_iovs = NULL; return accel_task; } @@ -1222,6 +1228,42 @@ accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_acce return 0; } +static void +accel_task_pull_data_cb(void *ctx, int status) +{ + struct spdk_accel_sequence *seq = ctx; + + assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); + if (spdk_likely(status == 0)) { + accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); + } else { + accel_sequence_set_fail(seq, status); + } + + accel_process_sequence(seq); +} + +static void +accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) +{ + int rc; + + assert(task->bounce.s.orig_iovs != NULL); + assert(task->bounce.s.orig_domain != NULL); + assert(task->bounce.s.orig_domain != g_accel_domain); + + rc = spdk_memory_domain_pull_data(task->bounce.s.orig_domain, + task->bounce.s.orig_domain_ctx, + task->bounce.s.orig_iovs, task->bounce.s.orig_iovcnt, + task->s.iovs, task->s.iovcnt, + accel_task_pull_data_cb, seq); + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n", + spdk_memory_domain_get_dma_device_id(task->bounce.s.orig_domain), rc); + accel_sequence_set_fail(seq, rc); + } +} + static void accel_process_sequence(struct spdk_accel_sequence *seq) { @@ -1265,6 +1307,10 @@ accel_process_sequence(struct spdk_accel_sequence *seq) accel_sequence_set_fail(seq, rc); break; } + if (task->bounce.s.orig_iovs != NULL) { + accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA); + break; + } accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); /* Fall through */ case ACCEL_SEQUENCE_STATE_EXEC_TASK: @@ -1285,6 +1331,10 @@ accel_process_sequence(struct spdk_accel_sequence *seq) accel_sequence_set_fail(seq, rc); } break; + case ACCEL_SEQUENCE_STATE_PULL_DATA: + accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); + accel_task_pull_data(seq, task); + break; case ACCEL_SEQUENCE_STATE_COMPLETE_TASK: TAILQ_REMOVE(&seq->tasks, task, seq_link); TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); @@ -1306,6 +1356,7 @@ accel_process_sequence(struct spdk_accel_sequence *seq) return; case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF: case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF: + case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA: case ACCEL_SEQUENCE_STATE_AWAIT_TASK: break; default: diff --git a/test/unit/lib/accel/accel.c/accel_ut.c b/test/unit/lib/accel/accel.c/accel_ut.c index 8f8bebef7..40334aad3 100644 --- a/test/unit/lib/accel/accel.c/accel_ut.c +++ b/test/unit/lib/accel/accel.c/accel_ut.c @@ -20,6 +20,12 @@ DEFINE_STUB(pmem_is_pmem, int, (const void *addr, size_t len), 0); DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL); #endif DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); +DEFINE_STUB(spdk_memory_domain_pull_data, int, + (struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, uint32_t siovcnt, + struct iovec *diov, uint32_t diovcnt, spdk_memory_domain_data_cpl_cb cpl_cb, + void *cpl_arg), 0); +DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, + (struct spdk_memory_domain *domain), "UT_DMA"); #ifdef SPDK_CONFIG_ISAL DEFINE_STUB_V(XTS_AES_128_enc, (uint8_t *k2, uint8_t *k1, uint8_t *tweak, uint64_t lba_size, const uint8_t *src, uint8_t *dst));