diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 412017b15..4d54fa0f2 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -46,6 +46,7 @@ #include "spdk/notify.h" #include "spdk/util.h" #include "spdk/trace.h" +#include "spdk/dma.h" #include "spdk/bdev_module.h" #include "spdk/log.h" @@ -322,6 +323,7 @@ struct spdk_bdev_desc { } callback; bool closed; bool write; + bool memory_domains_supported; pthread_mutex_t mutex; uint32_t refs; TAILQ_HEAD(, media_event_entry) pending_media_events; @@ -752,6 +754,12 @@ spdk_bdev_next_leaf(struct spdk_bdev *prev) return bdev; } +static inline bool +bdev_io_use_memory_domain(struct spdk_bdev_io *bdev_io) +{ + return bdev_io->internal.ext_opts && bdev_io->internal.ext_opts->memory_domain; +} + void spdk_bdev_io_set_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t len) { @@ -869,6 +877,8 @@ _bdev_io_pull_buffer_cpl(void *ctx, int rc) static void _bdev_io_pull_bounce_md_buf(struct spdk_bdev_io *bdev_io, void *md_buf, size_t len) { + int rc = 0; + /* save original md_buf */ bdev_io->internal.orig_md_iov.iov_base = bdev_io->u.bdev.md_buf; bdev_io->internal.orig_md_iov.iov_len = len; @@ -878,11 +888,26 @@ _bdev_io_pull_bounce_md_buf(struct spdk_bdev_io *bdev_io, void *md_buf, size_t l bdev_io->u.bdev.md_buf = md_buf; if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { - memcpy(md_buf, bdev_io->internal.orig_md_iov.iov_base, bdev_io->internal.orig_md_iov.iov_len); + if (bdev_io_use_memory_domain(bdev_io)) { + rc = spdk_memory_domain_pull_data(bdev_io->internal.ext_opts->memory_domain, + bdev_io->internal.ext_opts->memory_domain_ctx, + &bdev_io->internal.orig_md_iov, 1, + &bdev_io->internal.bounce_md_iov, 1, + bdev_io->internal.data_transfer_cpl, + bdev_io); + if (rc == 0) { + /* Continue to submit IO in completion callback */ + return; + } + SPDK_ERRLOG("Failed to pull data from memory domain %s, rc %d\n", + spdk_memory_domain_get_dma_device_id(bdev_io->internal.ext_opts->memory_domain), rc); + } else { + memcpy(md_buf, bdev_io->internal.orig_md_iov.iov_base, bdev_io->internal.orig_md_iov.iov_len); + } } assert(bdev_io->internal.data_transfer_cpl); - bdev_io->internal.data_transfer_cpl(bdev_io, 0); + bdev_io->internal.data_transfer_cpl(bdev_io, rc); } static void @@ -928,6 +953,8 @@ static void _bdev_io_pull_bounce_data_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t len, bdev_copy_bounce_buffer_cpl cpl_cb) { + int rc = 0; + bdev_io->internal.data_transfer_cpl = cpl_cb; /* save original iovec */ bdev_io->internal.orig_iovs = bdev_io->u.bdev.iovs; @@ -940,10 +967,26 @@ _bdev_io_pull_bounce_data_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t le bdev_io->u.bdev.iovs[0].iov_len = len; /* if this is write path, copy data from original buffer to bounce buffer */ if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { - _copy_iovs_to_buf(buf, len, bdev_io->internal.orig_iovs, bdev_io->internal.orig_iovcnt); + if (bdev_io_use_memory_domain(bdev_io)) { + rc = spdk_memory_domain_pull_data(bdev_io->internal.ext_opts->memory_domain, + bdev_io->internal.ext_opts->memory_domain_ctx, + bdev_io->internal.orig_iovs, + (uint32_t) bdev_io->internal.orig_iovcnt, + bdev_io->u.bdev.iovs, 1, + _bdev_io_pull_bounce_data_buf_done, + bdev_io); + if (rc == 0) { + /* Continue to submit IO in completion callback */ + return; + } + SPDK_ERRLOG("Failed to pull data from memory domain %s\n", + spdk_memory_domain_get_dma_device_id(bdev_io->internal.ext_opts->memory_domain)); + } else { + _copy_iovs_to_buf(buf, len, bdev_io->internal.orig_iovs, bdev_io->internal.orig_iovcnt); + } } - _bdev_io_pull_bounce_data_buf_done(bdev_io, 0); + _bdev_io_pull_bounce_data_buf_done(bdev_io, rc); } static void @@ -1122,19 +1165,38 @@ _bdev_io_complete_push_bounce_done(void *ctx, int rc) static inline void _bdev_io_push_bounce_md_buffer(struct spdk_bdev_io *bdev_io) { + int rc = 0; + /* do the same for metadata buffer */ if (spdk_unlikely(bdev_io->internal.orig_md_iov.iov_base != NULL)) { assert(spdk_bdev_is_md_separate(bdev_io->bdev)); if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ && bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) { - memcpy(bdev_io->internal.orig_md_iov.iov_base, bdev_io->u.bdev.md_buf, - bdev_io->internal.orig_md_iov.iov_len); + if (bdev_io_use_memory_domain(bdev_io)) { + /* If memory domain is used then we need to call async push function */ + rc = spdk_memory_domain_push_data(bdev_io->internal.ext_opts->memory_domain, + bdev_io->internal.ext_opts->memory_domain_ctx, + &bdev_io->internal.orig_md_iov, + (uint32_t)bdev_io->internal.orig_iovcnt, + &bdev_io->internal.bounce_md_iov, 1, + bdev_io->internal.data_transfer_cpl, + bdev_io); + if (rc == 0) { + /* Continue IO completion in async callback */ + return; + } + SPDK_ERRLOG("Failed to push md to memory domain %s\n", + spdk_memory_domain_get_dma_device_id(bdev_io->internal.ext_opts->memory_domain)); + } else { + memcpy(bdev_io->internal.orig_md_iov.iov_base, bdev_io->u.bdev.md_buf, + bdev_io->internal.orig_md_iov.iov_len); + } } } assert(bdev_io->internal.data_transfer_cpl); - bdev_io->internal.data_transfer_cpl(bdev_io, 0); + bdev_io->internal.data_transfer_cpl(bdev_io, rc); } static void @@ -1162,18 +1224,37 @@ _bdev_io_push_bounce_data_buffer_done(void *ctx, int rc) static inline void _bdev_io_push_bounce_data_buffer(struct spdk_bdev_io *bdev_io, bdev_copy_bounce_buffer_cpl cpl_cb) { + int rc = 0; + bdev_io->internal.data_transfer_cpl = cpl_cb; /* if this is read path, copy data from bounce buffer to original buffer */ if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ && bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) { - _copy_buf_to_iovs(bdev_io->internal.orig_iovs, - bdev_io->internal.orig_iovcnt, - bdev_io->internal.bounce_iov.iov_base, - bdev_io->internal.bounce_iov.iov_len); + if (bdev_io_use_memory_domain(bdev_io)) { + /* If memory domain is used then we need to call async push function */ + rc = spdk_memory_domain_push_data(bdev_io->internal.ext_opts->memory_domain, + bdev_io->internal.ext_opts->memory_domain_ctx, + bdev_io->internal.orig_iovs, + (uint32_t)bdev_io->internal.orig_iovcnt, + &bdev_io->internal.bounce_iov, 1, + _bdev_io_push_bounce_data_buffer_done, + bdev_io); + if (rc == 0) { + /* Continue IO completion in async callback */ + return; + } + SPDK_ERRLOG("Failed to push data to memory domain %s\n", + spdk_memory_domain_get_dma_device_id(bdev_io->internal.ext_opts->memory_domain)); + } else { + _copy_buf_to_iovs(bdev_io->internal.orig_iovs, + bdev_io->internal.orig_iovcnt, + bdev_io->internal.bounce_iov.iov_base, + bdev_io->internal.bounce_iov.iov_len); + } } - _bdev_io_push_bounce_data_buffer_done(bdev_io, 0); + _bdev_io_push_bounce_data_buffer_done(bdev_io, rc); } static void @@ -1239,6 +1320,28 @@ spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, u bdev_io_get_buf(bdev_io, len); } +static void +_bdev_memory_domain_get_io_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, + bool success) +{ + if (!success) { + SPDK_ERRLOG("Failed to get data buffer, completing IO\n"); + bdev_io_complete(bdev_io); + } else { + bdev_io_submit(bdev_io); + } +} + +static void +_bdev_memory_domain_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, + uint64_t len) +{ + assert(cb != NULL); + bdev_io->internal.get_buf_cb = cb; + + bdev_io_get_buf(bdev_io, len); +} + void spdk_bdev_io_get_aux_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_aux_buf_cb cb) { @@ -3989,6 +4092,33 @@ _bdev_io_check_md_buf(const struct iovec *iovs, const void *md_buf) return _is_buf_allocated(iovs) == (md_buf != NULL); } +static inline void +_bdev_io_copy_ext_opts(struct spdk_bdev_io *bdev_io, struct spdk_bdev_ext_io_opts *opts) +{ + struct spdk_bdev_ext_io_opts *opts_copy = &bdev_io->internal.ext_opts_copy; + + memcpy(opts_copy, opts, opts->size); + bdev_io->internal.ext_opts_copy.metadata = bdev_io->u.bdev.md_buf; + /* Save pointer to the copied ext_opts which will be used by bdev modules */ + bdev_io->u.bdev.ext_opts = opts_copy; +} + +static inline void +_bdev_io_ext_use_bounce_buffer(struct spdk_bdev_io *bdev_io) +{ + /* bdev doesn't support memory domains, thereby buffers in this IO request can't + * be accessed directly. It is needed to allocate buffers before issuing IO operation. + * For write operation we need to pull buffers from memory domain before submitting IO. + * Once read operation completes, we need to use memory_domain push functionality to + * update data in original memory domain IO buffer + * This IO request will go through a regular IO flow, so clear memory domains pointers in + * the copied ext_opts */ + bdev_io->internal.ext_opts_copy.memory_domain = NULL; + bdev_io->internal.ext_opts_copy.memory_domain_ctx = NULL; + _bdev_memory_domain_io_get_buf(bdev_io, _bdev_memory_domain_get_io_cb, + bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); +} + static int bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks, @@ -4093,7 +4223,6 @@ bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *c struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc); struct spdk_bdev_io *bdev_io; struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); - struct spdk_bdev_ext_io_opts *opts_copy; if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) { return -EINVAL; @@ -4116,13 +4245,17 @@ bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *c bdev_io->internal.ext_opts = opts; bdev_io->u.bdev.ext_opts = opts; - if (opts && copy_opts) { + if (opts) { + bool use_pull_push = opts->memory_domain && !desc->memory_domains_supported; + assert(opts->size <= sizeof(*opts)); - opts_copy = &bdev_io->internal.ext_opts_copy; - memcpy(opts_copy, opts, opts->size); - bdev_io->internal.ext_opts_copy.metadata = md_buf; - bdev_io->internal.ext_opts = opts_copy; - bdev_io->u.bdev.ext_opts = opts_copy; + if (copy_opts || use_pull_push) { + _bdev_io_copy_ext_opts(bdev_io, opts); + if (use_pull_push) { + _bdev_io_ext_use_bounce_buffer(bdev_io); + return 0; + } + } } bdev_io_submit(bdev_io); @@ -4170,6 +4303,10 @@ spdk_bdev_readv_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel * if (spdk_unlikely(!opts->size || opts->size > sizeof(struct spdk_bdev_ext_io_opts))) { return -EINVAL; } + if (spdk_unlikely(opts->memory_domain && !(iov && iov[0].iov_base))) { + /* When memory domain is used, the user must provide data buffers */ + return -EINVAL; + } md = opts->metadata; } @@ -4279,7 +4416,6 @@ bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel * struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc); struct spdk_bdev_io *bdev_io; struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); - struct spdk_bdev_ext_io_opts *opts_copy; if (!desc->write) { return -EBADF; @@ -4306,13 +4442,17 @@ bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel * bdev_io->internal.ext_opts = opts; bdev_io->u.bdev.ext_opts = opts; - if (opts && copy_opts) { + if (opts) { + bool use_pull_push = opts->memory_domain && !desc->memory_domains_supported; + assert(opts->size <= sizeof(*opts)); - opts_copy = &bdev_io->internal.ext_opts_copy; - memcpy(opts_copy, opts, opts->size); - bdev_io->internal.ext_opts_copy.metadata = md_buf; - bdev_io->internal.ext_opts = opts_copy; - bdev_io->u.bdev.ext_opts = opts_copy; + if (copy_opts || use_pull_push) { + _bdev_io_copy_ext_opts(bdev_io, opts); + if (use_pull_push) { + _bdev_io_ext_use_bounce_buffer(bdev_io); + return 0; + } + } } bdev_io_submit(bdev_io); @@ -4377,6 +4517,10 @@ spdk_bdev_writev_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel if (spdk_unlikely(!opts->size || opts->size > sizeof(struct spdk_bdev_ext_io_opts))) { return -EINVAL; } + if (spdk_unlikely(opts->memory_domain && !(iov && iov[0].iov_base))) { + /* When memory domain is used, the user must provide data buffers */ + return -EINVAL; + } md = opts->metadata; } @@ -6282,6 +6426,7 @@ spdk_bdev_open_ext(const char *bdev_name, bool write, spdk_bdev_event_cb_t event TAILQ_INIT(&desc->pending_media_events); TAILQ_INIT(&desc->free_media_events); + desc->memory_domains_supported = spdk_bdev_get_memory_domains(bdev, NULL, 0) > 0; desc->callback.event_fn = event_cb; desc->callback.ctx = event_ctx; pthread_mutex_init(&desc->mutex, NULL); diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index c849e0608..f66d19281 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -83,7 +83,7 @@ DEPDIRS-net := log util $(JSON_LIBS) DEPDIRS-notify := log util $(JSON_LIBS) DEPDIRS-trace := log util $(JSON_LIBS) -DEPDIRS-bdev := log util thread $(JSON_LIBS) notify trace +DEPDIRS-bdev := log util thread $(JSON_LIBS) notify trace dma DEPDIRS-blobfs := log thread blob trace DEPDIRS-event := log util thread $(JSON_LIBS) trace init DEPDIRS-init := jsonrpc json log rpc thread util diff --git a/test/dma/test_dma/test_dma.c b/test/dma/test_dma/test_dma.c index 6119ac6e5..46e3cfe3e 100644 --- a/test/dma/test_dma/test_dma.c +++ b/test/dma/test_dma/test_dma.c @@ -81,6 +81,11 @@ struct dma_test_task { TAILQ_ENTRY(dma_test_task) link; }; +struct dma_test_data_cpl_ctx { + spdk_memory_domain_data_cpl_cb data_cpl; + void *data_cpl_arg; +}; + TAILQ_HEAD(, dma_test_task) g_tasks = TAILQ_HEAD_INITIALIZER(g_tasks); /* User's input */ @@ -274,6 +279,55 @@ dma_test_task_is_read(struct dma_test_task *task) return false; } +static void +dma_test_data_cpl(void *ctx) +{ + struct dma_test_data_cpl_ctx *cpl_ctx = ctx; + + cpl_ctx->data_cpl(cpl_ctx->data_cpl_arg, 0); + free(cpl_ctx); +} + +static int +dma_test_copy_memory(struct dma_test_req *req, struct iovec *dst_iov, uint32_t dst_iovcnt, + struct iovec *src_iov, uint32_t src_iovcnt, spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + struct dma_test_data_cpl_ctx *cpl_ctx; + + cpl_ctx = calloc(1, sizeof(*cpl_ctx)); + if (!cpl_ctx) { + return -ENOMEM; + } + + cpl_ctx->data_cpl = cpl_cb; + cpl_ctx->data_cpl_arg = cpl_cb_arg; + + spdk_iovcpy(src_iov, src_iovcnt, dst_iov, dst_iovcnt); + spdk_thread_send_msg(req->task->thread, dma_test_data_cpl, cpl_ctx); + + return 0; +} + +static int dma_test_push_memory_cb(struct spdk_memory_domain *dst_domain, + void *dst_domain_ctx, + struct iovec *dst_iov, uint32_t dst_iovcnt, struct iovec *src_iov, uint32_t src_iovcnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + struct dma_test_req *req = dst_domain_ctx; + + return dma_test_copy_memory(req, dst_iov, dst_iovcnt, src_iov, src_iovcnt, cpl_cb, cpl_cb_arg); +} + +static int dma_test_pull_memory_cb(struct spdk_memory_domain *src_domain, + void *src_domain_ctx, + struct iovec *src_iov, uint32_t src_iovcnt, struct iovec *dst_iov, uint32_t dst_iovcnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + struct dma_test_req *req = src_domain_ctx; + + return dma_test_copy_memory(req, dst_iov, dst_iovcnt, src_iov, src_iovcnt, cpl_cb, cpl_cb_arg); +} + static int dma_test_translate_memory_cb(struct spdk_memory_domain *src_domain, void *src_domain_ctx, struct spdk_memory_domain *dst_domain, struct spdk_memory_domain_translation_ctx *dst_domain_ctx, @@ -666,6 +720,8 @@ dma_test_start(void *arg) return; } spdk_memory_domain_set_translation(g_domain, dma_test_translate_memory_cb); + spdk_memory_domain_set_pull(g_domain, dma_test_pull_memory_cb); + spdk_memory_domain_set_push(g_domain, dma_test_push_memory_cb); SPDK_ENV_FOREACH_CORE(i) { rc = allocate_task(i, g_bdev_name); diff --git a/test/nvmf/host/dma.sh b/test/nvmf/host/dma.sh index 7597a66e3..f83f01b69 100755 --- a/test/nvmf/host/dma.sh +++ b/test/nvmf/host/dma.sh @@ -14,6 +14,50 @@ MALLOC_BLOCK_SIZE=512 subsystem="0" rpc_py="$rootdir/scripts/rpc.py" +function gen_malloc_json() { + jq . <<- JSON + { + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_nvme_set_options", + "params": { + "action_on_timeout": "none", + "timeout_us": 0, + "retry_count": 4, + "arbitration_burst": 0, + "low_priority_weight": 0, + "medium_priority_weight": 0, + "high_priority_weight": 0, + "nvme_adminq_poll_period_us": 10000, + "keep_alive_timeout_ms" : 10000, + "nvme_ioq_poll_period_us": 0, + "io_queue_requests": 0, + "delay_cmd_submit": true + } + }, + { + "method": "bdev_malloc_create", + "params": { + "name": "Malloc0", + "num_blocks": 131072, + "block_size": 512, + "uuid": "e1c24cb1-dd44-4be6-8d67-de92a332013f", + "optimal_io_boundary": 2 + } + }, + { + "method": "bdev_wait_for_examine" + } + ] + } + ] + } + JSON +} + nvmftestinit nvmfappstart -m 0x3 @@ -27,6 +71,9 @@ $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$subsystem -t $TEST # test_dma doesn't use RPC, but we change the sock path since nvmf target is already using the default RPC sock "$rootdir/test/dma/test_dma/test_dma" -q 16 -o 4096 -w randrw -M 70 -t 5 -m 0xc --json <(gen_nvmf_target_json $subsystem) -b "Nvme${subsystem}n1" -f -r /var/tmp/dma.sock +# test data pull/push with split against local malloc +"$rootdir/test/dma/test_dma/test_dma" -q 16 -o 4096 -w randrw -M 70 -t 5 -m 0xc --json <(gen_malloc_json) -b "Malloc0" -r /var/tmp/dma.sock + trap - SIGINT SIGTERM EXIT nvmftestfini diff --git a/test/unit/lib/bdev/bdev.c/bdev_ut.c b/test/unit/lib/bdev/bdev.c/bdev_ut.c index 1714b0965..7a83d9eaa 100644 --- a/test/unit/lib/bdev/bdev.c/bdev_ut.c +++ b/test/unit/lib/bdev/bdev.c/bdev_ut.c @@ -45,6 +45,37 @@ DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); +DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, (struct spdk_memory_domain *domain), + "test_domain"); +DEFINE_STUB(spdk_memory_domain_get_dma_device_type, enum spdk_dma_device_type, + (struct spdk_memory_domain *domain), 0); + +static bool g_memory_domain_pull_data_called; +static bool g_memory_domain_push_data_called; + +DEFINE_RETURN_MOCK(spdk_memory_domain_pull_data, int); +int +spdk_memory_domain_pull_data(struct spdk_memory_domain *src_domain, void *src_domain_ctx, + struct iovec *src_iov, uint32_t src_iov_cnt, struct iovec *dst_iov, uint32_t dst_iov_cnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + g_memory_domain_pull_data_called = true; + HANDLE_RETURN_MOCK(spdk_memory_domain_pull_data); + cpl_cb(cpl_cb_arg, 0); + return 0; +} + +DEFINE_RETURN_MOCK(spdk_memory_domain_push_data, int); +int +spdk_memory_domain_push_data(struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, + struct iovec *dst_iov, uint32_t dst_iovcnt, struct iovec *src_iov, uint32_t src_iovcnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + g_memory_domain_push_data_called = true; + HANDLE_RETURN_MOCK(spdk_memory_domain_push_data); + cpl_cb(cpl_cb_arg, 0); + return 0; +} int g_status; int g_count; @@ -238,8 +269,8 @@ stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) if (expected_io->md_buf != NULL) { CU_ASSERT(expected_io->md_buf == bdev_io->u.bdev.md_buf); - if (bdev_io->internal.ext_opts) { - CU_ASSERT(expected_io->md_buf == bdev_io->internal.ext_opts->metadata); + if (bdev_io->u.bdev.ext_opts) { + CU_ASSERT(expected_io->md_buf == bdev_io->u.bdev.ext_opts->metadata); } } @@ -4900,7 +4931,8 @@ bdev_writev_readv_ext(void) struct spdk_bdev *bdev; struct spdk_bdev_desc *desc = NULL; struct spdk_io_channel *io_ch; - struct iovec iov = { .iov_base = (void *)0xbaaddead, .iov_len = 0x1000 }; + char io_buf[512]; + struct iovec iov = { .iov_base = io_buf, .iov_len = 512 }; struct ut_expected_io *expected_io; struct spdk_bdev_ext_io_opts ext_io_opts = { .metadata = (void *)0xFF000000, @@ -4921,6 +4953,7 @@ bdev_writev_readv_ext(void) io_ch = spdk_bdev_get_io_channel(desc); CU_ASSERT(io_ch != NULL); + /* Test 1, Simple test */ g_io_done = false; expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 32, 14, 1); expected_io->md_buf = ext_io_opts.metadata; @@ -4951,7 +4984,7 @@ bdev_writev_readv_ext(void) stub_complete_io(1); CU_ASSERT(g_io_done == true); - /* Test invalid ext_opts size */ + /* Test 2, invalid ext_opts size */ ext_io_opts.size = 0; rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); CU_ASSERT(rc != 0); @@ -4964,7 +4997,7 @@ bdev_writev_readv_ext(void) rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); CU_ASSERT(rc != 0); - /* Check that IO request with ext_opts and metadata is split correctly + /* Test 3, Check that IO request with ext_opts and metadata is split correctly * Offset 14, length 8, payload 0xF000 * Child - Offset 14, length 2, payload 0xF000 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 @@ -5079,6 +5112,40 @@ bdev_writev_readv_ext(void) CU_ASSERT(g_io_done == true); CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); + /* Test 4, Verify data pull/push + * bdev doens't support memory domains, so buffers from bdev memory pool will be used */ + ext_io_opts.memory_domain = (struct spdk_memory_domain *)0xdeadbeef; + + g_io_done = false; + expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 32, 14, 1); + ut_expected_io_set_iov(expected_io, 0, iov.iov_base, iov.iov_len); + expected_io->ext_io_opts = &ext_io_opts; + TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); + + rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); + + CU_ASSERT(rc == 0); + CU_ASSERT(g_io_done == false); + CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); + stub_complete_io(1); + CU_ASSERT(g_memory_domain_push_data_called == true); + CU_ASSERT(g_io_done == true); + + g_io_done = false; + expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 1); + ut_expected_io_set_iov(expected_io, 0, iov.iov_base, iov.iov_len); + expected_io->ext_io_opts = &ext_io_opts; + TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); + + rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); + + CU_ASSERT(rc == 0); + CU_ASSERT(g_memory_domain_pull_data_called == true); + CU_ASSERT(g_io_done == false); + CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); + stub_complete_io(1); + CU_ASSERT(g_io_done == true); + spdk_put_io_channel(io_ch); spdk_bdev_close(desc); free_bdev(bdev); diff --git a/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c b/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c index 693720ad0..393939b9a 100644 --- a/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c +++ b/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c @@ -48,6 +48,34 @@ DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); DEFINE_STUB_V(spdk_scsi_nvme_translate, (const struct spdk_bdev_io *bdev_io, int *sc, int *sk, int *asc, int *ascq)); +DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, (struct spdk_memory_domain *domain), + "test_domain"); +DEFINE_STUB(spdk_memory_domain_get_dma_device_type, enum spdk_dma_device_type, + (struct spdk_memory_domain *domain), 0); + +DEFINE_RETURN_MOCK(spdk_memory_domain_pull_data, int); +int +spdk_memory_domain_pull_data(struct spdk_memory_domain *src_domain, void *src_domain_ctx, + struct iovec *src_iov, uint32_t src_iov_cnt, struct iovec *dst_iov, uint32_t dst_iov_cnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + HANDLE_RETURN_MOCK(spdk_memory_domain_pull_data); + + cpl_cb(cpl_cb_arg, 0); + return 0; +} + +DEFINE_RETURN_MOCK(spdk_memory_domain_push_data, int); +int +spdk_memory_domain_push_data(struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, + struct iovec *dst_iov, uint32_t dst_iovcnt, struct iovec *src_iov, uint32_t src_iovcnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + HANDLE_RETURN_MOCK(spdk_memory_domain_push_data); + + cpl_cb(cpl_cb_arg, 0); + return 0; +} struct ut_bdev { struct spdk_bdev bdev; diff --git a/test/unit/lib/bdev/part.c/part_ut.c b/test/unit/lib/bdev/part.c/part_ut.c index 4e3c52889..7b21224d6 100644 --- a/test/unit/lib/bdev/part.c/part_ut.c +++ b/test/unit/lib/bdev/part.c/part_ut.c @@ -45,6 +45,34 @@ DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); +DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, (struct spdk_memory_domain *domain), + "test_domain"); +DEFINE_STUB(spdk_memory_domain_get_dma_device_type, enum spdk_dma_device_type, + (struct spdk_memory_domain *domain), 0); + +DEFINE_RETURN_MOCK(spdk_memory_domain_pull_data, int); +int +spdk_memory_domain_pull_data(struct spdk_memory_domain *src_domain, void *src_domain_ctx, + struct iovec *src_iov, uint32_t src_iov_cnt, struct iovec *dst_iov, uint32_t dst_iov_cnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + HANDLE_RETURN_MOCK(spdk_memory_domain_pull_data); + + cpl_cb(cpl_cb_arg, 0); + return 0; +} + +DEFINE_RETURN_MOCK(spdk_memory_domain_push_data, int); +int +spdk_memory_domain_push_data(struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, + struct iovec *dst_iov, uint32_t dst_iovcnt, struct iovec *src_iov, uint32_t src_iovcnt, + spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) +{ + HANDLE_RETURN_MOCK(spdk_memory_domain_push_data); + + cpl_cb(cpl_cb_arg, 0); + return 0; +} static void _part_cleanup(struct spdk_bdev_part *part)