diff --git a/include/spdk/dif.h b/include/spdk/dif.h index eb1d7351b..20242d8b5 100644 --- a/include/spdk/dif.h +++ b/include/spdk/dif.h @@ -150,6 +150,34 @@ int spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, int spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk); +/** + * Copy data and generate DIF for extended LBA payload. + * + * \param iovs iovec array describing the LBA payload. + * \param iovcnt Number of elements in the iovec array. + * \param bounce_iov A contiguous buffer forming extended LBA payload. + * \param num_blocks Number of blocks of the LBA payload. + * \param ctx DIF context. + * + * \return 0 on success and negated errno otherwise. + */ +int spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx); + +/** + * Verify DIF and copy data for extended LBA payload. + * + * \param iovs iovec array describing the LBA payload. + * \param iovcnt Number of elements in the iovec array. + * \param bounce_iov A contiguous buffer forming extended LBA payload. + * \param num_blocks Number of blocks of the LBA payload. + * \param ctx DIF context. + * + * \return 0 on success and negated errno otherwise. + */ +int spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx); + /** * Inject bit flip error to extended LBA payload. * diff --git a/lib/util/dif.c b/lib/util/dif.c index 6bfba6b31..a8b671706 100644 --- a/lib/util/dif.c +++ b/lib/util/dif.c @@ -577,6 +577,268 @@ spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, } } +static void +dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + struct _iov_iter src_iter, dst_iter; + uint32_t offset_blocks, data_block_size; + void *src, *dst; + uint16_t guard; + + offset_blocks = 0; + _iov_iter_init(&src_iter, iovs, iovcnt); + _iov_iter_init(&dst_iter, bounce_iov, 1); + + data_block_size = ctx->block_size - ctx->md_size; + + while (offset_blocks < num_blocks && + _iov_iter_cont(&src_iter) && _iov_iter_cont(&dst_iter)) { + + _iov_iter_get_buf(&src_iter, &src, NULL); + _iov_iter_get_buf(&dst_iter, &dst, NULL); + + guard = 0; + if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(0, dst, src, data_block_size); + guard = spdk_crc16_t10dif(guard, dst + data_block_size, + ctx->guard_interval - data_block_size); + } else { + memcpy(dst, src, data_block_size); + } + + _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); + + _iov_iter_advance(&src_iter, data_block_size); + _iov_iter_advance(&dst_iter, ctx->block_size); + offset_blocks++; + } +} + +static void +_dif_generate_copy_split(struct _iov_iter *src_iter, struct _iov_iter *dst_iter, + uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_in_block, src_len, data_block_size; + uint16_t guard; + void *src, *dst; + + _iov_iter_get_buf(dst_iter, &dst, NULL); + + data_block_size = ctx->block_size - ctx->md_size; + + guard = 0; + offset_in_block = 0; + + while (offset_in_block < data_block_size && _iov_iter_cont(src_iter)) { + /* Compute CRC over split logical block data and copy + * data to bounce buffer. + */ + _iov_iter_get_buf(src_iter, &src, &src_len); + src_len = spdk_min(src_len, data_block_size - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block, + src, src_len); + } else { + memcpy(dst + offset_in_block, src, src_len); + } + + _iov_iter_advance(src_iter, src_len); + offset_in_block += src_len; + } + + if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, dst + data_block_size, + ctx->guard_interval - data_block_size); + } + + _iov_iter_advance(dst_iter, ctx->block_size); + + _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); +} + +static void +dif_generate_copy_split(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + struct _iov_iter src_iter, dst_iter; + uint32_t offset_blocks; + + offset_blocks = 0; + _iov_iter_init(&src_iter, iovs, iovcnt); + _iov_iter_init(&dst_iter, bounce_iov, 1); + + while (offset_blocks < num_blocks && + _iov_iter_cont(&src_iter) && _iov_iter_cont(&dst_iter)) { + _dif_generate_copy_split(&src_iter, &dst_iter, offset_blocks, ctx); + offset_blocks++; + } +} + +int +spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t data_block_size; + + data_block_size = ctx->block_size - ctx->md_size; + + if (!_are_iovs_valid(iovs, iovcnt, data_block_size * num_blocks) || + !_are_iovs_valid(bounce_iov, 1, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_are_iovs_bytes_multiple(iovs, iovcnt, data_block_size)) { + dif_generate_copy(iovs, iovcnt, bounce_iov, num_blocks, ctx); + } else { + dif_generate_copy_split(iovs, iovcnt, bounce_iov, num_blocks, ctx); + } + + return 0; +} + +static int +dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + struct _iov_iter src_iter, dst_iter; + uint32_t offset_blocks, data_block_size; + void *src, *dst; + int rc; + uint16_t guard; + + offset_blocks = 0; + _iov_iter_init(&src_iter, bounce_iov, 1); + _iov_iter_init(&dst_iter, iovs, iovcnt); + + data_block_size = ctx->block_size - ctx->md_size; + + while (offset_blocks < num_blocks && _iov_iter_cont(&src_iter) && + _iov_iter_cont(&dst_iter)) { + + _iov_iter_get_buf(&src_iter, &src, NULL); + _iov_iter_get_buf(&dst_iter, &dst, NULL); + + guard = 0; + if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(0, dst, src, data_block_size); + guard = spdk_crc16_t10dif(guard, src + data_block_size, + ctx->guard_interval - data_block_size); + } else { + memcpy(dst, src, data_block_size); + } + + rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, NULL); + if (rc != 0) { + return rc; + } + + _iov_iter_advance(&src_iter, ctx->block_size); + _iov_iter_advance(&dst_iter, data_block_size); + offset_blocks++; + } + + return 0; +} + +static int +_dif_verify_copy_split(struct _iov_iter *src_iter, struct _iov_iter *dst_iter, + uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_in_block, dst_len, data_block_size; + uint16_t guard; + void *src, *dst; + + _iov_iter_get_buf(src_iter, &src, NULL); + + data_block_size = ctx->block_size - ctx->md_size; + + guard = 0; + offset_in_block = 0; + + while (offset_in_block < data_block_size) { + /* Compute CRC over split logical block data and copy + * data to bounce buffer. + */ + _iov_iter_get_buf(dst_iter, &dst, &dst_len); + dst_len = spdk_min(dst_len, data_block_size - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(guard, dst, + src + offset_in_block, dst_len); + } else { + memcpy(dst, src + offset_in_block, dst_len); + } + + _iov_iter_advance(dst_iter, dst_len); + offset_in_block += dst_len; + } + + if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, src + data_block_size, + ctx->guard_interval - data_block_size); + } + + _iov_iter_advance(src_iter, ctx->block_size); + + return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, NULL); +} + +static int +dif_verify_copy_split(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + struct _iov_iter src_iter, dst_iter; + uint32_t offset_blocks; + int rc; + + offset_blocks = 0; + _iov_iter_init(&src_iter, bounce_iov, 1); + _iov_iter_init(&dst_iter, iovs, iovcnt); + + while (offset_blocks < num_blocks && + _iov_iter_cont(&src_iter) && _iov_iter_cont(&dst_iter)) { + rc = _dif_verify_copy_split(&src_iter, &dst_iter, offset_blocks, ctx); + if (rc != 0) { + return rc; + } + offset_blocks++; + } + + return 0; +} + +int +spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t data_block_size; + + data_block_size = ctx->block_size - ctx->md_size; + + if (!_are_iovs_valid(iovs, iovcnt, data_block_size * num_blocks) || + !_are_iovs_valid(bounce_iov, 1, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec arrays are not valid\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_are_iovs_bytes_multiple(iovs, iovcnt, data_block_size)) { + return dif_verify_copy(iovs, iovcnt, bounce_iov, num_blocks, ctx); + } else { + return dif_verify_copy_split(iovs, iovcnt, bounce_iov, num_blocks, ctx); + } +} + static void _bit_flip(uint8_t *buf, uint32_t flip_bit) { diff --git a/test/unit/lib/util/dif.c/dif_ut.c b/test/unit/lib/util/dif.c/dif_ut.c index d5cbd2641..236abf211 100644 --- a/test/unit/lib/util/dif.c/dif_ut.c +++ b/test/unit/lib/util/dif.c/dif_ut.c @@ -728,6 +728,170 @@ dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test(void) _iov_free_buf(&iovs[1]); } +static void +dif_copy_gen_and_verify(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t block_size, uint32_t md_size, uint32_t num_blocks, + bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, + uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag) +{ + struct spdk_dif_ctx ctx = {}; + int rc; + + rc = ut_data_pattern_generate(iovs, iovcnt, block_size - md_size, 0, num_blocks); + CU_ASSERT(rc == 0); + + rc = spdk_dif_ctx_init(&ctx, block_size, md_size, dif_loc, dif_type, dif_flags, + init_ref_tag, apptag_mask, app_tag); + CU_ASSERT(rc == 0); + + rc = spdk_dif_generate_copy(iovs, iovcnt, bounce_iov, num_blocks, &ctx); + CU_ASSERT(rc == 0); + + rc = spdk_dif_verify_copy(iovs, iovcnt, bounce_iov, num_blocks, &ctx); + CU_ASSERT(rc == 0); + + rc = ut_data_pattern_verify(iovs, iovcnt, block_size - md_size, 0, num_blocks); + CU_ASSERT(rc == 0); +} + +static void +dif_copy_sec_512_md_8_prchk_0_single_iov(void) +{ + struct iovec iov, bounce_iov; + + _iov_alloc_buf(&iov, 512 * 4); + _iov_alloc_buf(&bounce_iov, (512 + 8) * 4); + + dif_copy_gen_and_verify(&iov, 1, &bounce_iov, 512 + 8, 8, 4, + false, SPDK_DIF_TYPE1, 0, 0, 0, 0); + dif_copy_gen_and_verify(&iov, 1, &bounce_iov, 512 + 8, 8, 4, + true, SPDK_DIF_TYPE1, 0, 0, 0, 0); + + _iov_free_buf(&iov); + _iov_free_buf(&bounce_iov); +} + +static void +dif_copy_sec_512_md_8_prchk_0_1_2_4_multi_iovs(void) +{ + struct iovec iovs[4], bounce_iov; + int i, num_blocks; + + num_blocks = 0; + + for (i = 0; i < 4; i++) { + _iov_alloc_buf(&iovs[i], 512 * (i + 1)); + num_blocks += i + 1; + } + + _iov_alloc_buf(&bounce_iov, (512 + 8) * num_blocks); + + dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks, + false, SPDK_DIF_TYPE1, 0, 22, 0xFFFF, 0x22); + + dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks, + false, SPDK_DIF_TYPE1, SPDK_DIF_GUARD_CHECK, 22, 0xFFFF, 0x22); + + dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks, + false, SPDK_DIF_TYPE1, SPDK_DIF_APPTAG_CHECK, 22, 0xFFFF, 0x22); + + dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks, + false, SPDK_DIF_TYPE1, SPDK_DIF_REFTAG_CHECK, 22, 0xFFFF, 0x22); + + for (i = 0; i < 4; i++) { + _iov_free_buf(&iovs[i]); + } + _iov_free_buf(&bounce_iov); +} + +static void +dif_copy_sec_4096_md_128_prchk_7_multi_iovs(void) +{ + struct iovec iovs[4], bounce_iov; + uint32_t dif_flags; + int i, num_blocks; + + dif_flags = SPDK_DIF_GUARD_CHECK | SPDK_DIF_APPTAG_CHECK | SPDK_DIF_REFTAG_CHECK; + + num_blocks = 0; + + for (i = 0; i < 4; i++) { + _iov_alloc_buf(&iovs[i], 4096 * (i + 1)); + num_blocks += i + 1; + } + + _iov_alloc_buf(&bounce_iov, (4096 + 128) * num_blocks); + + dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 4096 + 128, 128, num_blocks, + false, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22); + dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 4096 + 128, 128, num_blocks, + true, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22); + + for (i = 0; i < 4; i++) { + _iov_free_buf(&iovs[i]); + } + _iov_free_buf(&bounce_iov); +} + +static void +dif_copy_sec_512_md_8_prchk_7_multi_iovs_split_data(void) +{ + struct iovec iovs[2], bounce_iov; + uint32_t dif_flags; + + dif_flags = SPDK_DIF_GUARD_CHECK | SPDK_DIF_APPTAG_CHECK | SPDK_DIF_REFTAG_CHECK; + + _iov_alloc_buf(&iovs[0], 256); + _iov_alloc_buf(&iovs[1], 256); + + _iov_alloc_buf(&bounce_iov, 512 + 8); + + dif_copy_gen_and_verify(iovs, 2, &bounce_iov, 512 + 8, 8, 1, + false, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22); + + _iov_free_buf(&iovs[0]); + _iov_free_buf(&iovs[1]); + _iov_free_buf(&bounce_iov); +} + +static void +dif_copy_sec_512_md_8_prchk_7_multi_iovs_complex_splits(void) +{ + struct iovec iovs[6], bounce_iov; + uint32_t dif_flags; + int i; + + dif_flags = SPDK_DIF_GUARD_CHECK | SPDK_DIF_APPTAG_CHECK | SPDK_DIF_REFTAG_CHECK; + + /* data[0][255:0] */ + _iov_alloc_buf(&iovs[0], 256); + + /* data[0][511:256], data[1][255:0] */ + _iov_alloc_buf(&iovs[1], 256 + 256); + + /* data[1][382:256] */ + _iov_alloc_buf(&iovs[2], 128); + + /* data[1][383] */ + _iov_alloc_buf(&iovs[3], 1); + + /* data[1][510:384] */ + _iov_alloc_buf(&iovs[4], 126); + + /* data[1][511], data[2][511:0], data[3][511:0] */ + _iov_alloc_buf(&iovs[5], 1 + 512 * 2); + + _iov_alloc_buf(&bounce_iov, (512 + 8) * 4); + + dif_copy_gen_and_verify(iovs, 6, &bounce_iov, 512 + 8, 8, 4, + true, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22); + + for (i = 0; i < 6; i++) { + _iov_free_buf(&iovs[i]); + } + _iov_free_buf(&bounce_iov); +} + int main(int argc, char **argv) { @@ -781,7 +945,17 @@ main(int argc, char **argv) CU_add_test(suite, "dif_sec_4096_md_128_inject_1_2_4_8__multi_iovs_split_apptag_test", dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_apptag_test) == NULL || CU_add_test(suite, "dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test", - dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test) == NULL + dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test) == NULL || + CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_0_single_iov", + dif_copy_sec_512_md_8_prchk_0_single_iov) == NULL || + CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_0_1_2_4_multi_iovs", + dif_copy_sec_512_md_8_prchk_0_1_2_4_multi_iovs) == NULL || + CU_add_test(suite, "dif_copy_sec_4096_md_128_prchk_7_multi_iovs", + dif_copy_sec_4096_md_128_prchk_7_multi_iovs) == NULL || + CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_7_multi_iovs_split_data", + dif_copy_sec_512_md_8_prchk_7_multi_iovs_split_data) == NULL || + CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_7_multi_iovs_complex_splits", + dif_copy_sec_512_md_8_prchk_7_multi_iovs_complex_splits) == NULL ) { CU_cleanup_registry(); return CU_get_error();