ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bdev/error: Support injecting data corruption into I/O buffer

Browse Source 
Support injecting data corruption for processing read/write I/O.

Add two parameters, corrupt_offset and corrupt_value, and an error
type corrupt_data.

For write I/O, inject data corruption before submitting write I/O
to the underlying bdev. For read I/O, inject data corruption after
read I/O successfully returned.

Cause the data corruption by xoring the value at the corrupt_offset
with the corrupt_value. corrupt_value is required to be non-zero to
cause data corruption necessarily.

Signed-off-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I67d8d252b06d7a221413e74996f7f894e6519556
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15028
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
Community-CI: Mellanox Build Bot
This commit is contained in:
Shuhei Matsumoto 2022-10-28 11:01:10 +09:00 committed by Tomasz Zawadzki
parent e0daee9840
commit 3d2a3ee4df
6 changed files with 86 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/jsonrpc.md
View File

@ -4596,8 +4596,10 @@ Name | Optional | Type | Description
----------------------- | -------- | ----------- | -----------
name | Required | string | Name of the error injection bdev
io_type | Required | string | io type 'clear' 'read' 'write' 'unmap' 'flush' 'all'
error_type | Required | string | error type 'failure' 'pending'
error_type | Required | string | error type 'failure' 'pending' 'corrupt_data'
num | Optional | int | the number of commands you want to fail.(default:1)
corrupt_offset | Optional | int | the offset in bytes to xor with corrupt_value
corrupt_value | Optional | int | the value for xor (1-255, 0 is invalid)
#### Example

59
module/bdev/error/vbdev_error.c
View File

@ -30,6 +30,8 @@ static TAILQ_HEAD(, spdk_vbdev_error_config) g_error_config
struct vbdev_error_info {
uint32_t error_type;
uint32_t error_num;
uint64_t corrupt_offset;
uint8_t corrupt_value;
};
/* Context for each error bdev */
@ -81,6 +83,14 @@ vbdev_error_inject_error(char *name, const struct vbdev_error_inject_opts *opts)
uint32_t i;
int rc = 0;
if (opts->error_type == VBDEV_IO_CORRUPT_DATA) {
if (opts->corrupt_value == 0) {
/* If corrupt_value is 0, XOR cannot cause data corruption. */
SPDK_ERRLOG("corrupt_value should be non-zero.\n");
return -EINVAL;
}
}
pthread_mutex_lock(&g_vbdev_error_mutex);
rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &desc);
@ -109,6 +119,8 @@ vbdev_error_inject_error(char *name, const struct vbdev_error_inject_opts *opts)
for (i = 0; i < SPDK_COUNTOF(error_disk->error_vector); i++) {
error_disk->error_vector[i].error_type = opts->error_type;
error_disk->error_vector[i].error_num = opts->error_num;
error_disk->error_vector[i].corrupt_offset = opts->corrupt_offset;
error_disk->error_vector[i].corrupt_value = opts->corrupt_value;
}
} else if (0 == opts->io_type) {
for (i = 0; i < SPDK_COUNTOF(error_disk->error_vector); i++) {
@ -117,6 +129,8 @@ vbdev_error_inject_error(char *name, const struct vbdev_error_inject_opts *opts)
} else {
error_disk->error_vector[opts->io_type].error_type = opts->error_type;
error_disk->error_vector[opts->io_type].error_num = opts->error_num;
error_disk->error_vector[opts->io_type].corrupt_offset = opts->corrupt_offset;
error_disk->error_vector[opts->io_type].corrupt_value = opts->corrupt_value;
}
exit:
@ -156,10 +170,46 @@ vbdev_error_get_error_type(struct error_disk *error_disk, uint32_t io_type)
return 0;
}
static void
vbdev_error_corrupt_io_data(struct spdk_bdev_io *bdev_io, uint64_t corrupt_offset,
uint8_t corrupt_value)
{
uint8_t *buf;
int i;
if (bdev_io->u.bdev.iovs == NULL || bdev_io->u.bdev.iovs[0].iov_base == NULL) {
return;
}
for (i = 0; i < bdev_io->u.bdev.iovcnt; i++) {
if (bdev_io->u.bdev.iovs[i].iov_len > corrupt_offset) {
buf = (uint8_t *)bdev_io->u.bdev.iovs[i].iov_base;
buf[corrupt_offset] ^= corrupt_value;
break;
}
corrupt_offset -= bdev_io->u.bdev.iovs[i].iov_len;
}
}
static void
vbdev_error_complete_request(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED;
struct error_disk *error_disk = bdev_io->bdev->ctxt;
uint32_t error_type;
if (success && bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
error_type = vbdev_error_get_error_type(error_disk, bdev_io->type);
if (error_type == VBDEV_IO_CORRUPT_DATA) {
error_disk->error_vector[bdev_io->type].error_num--;
vbdev_error_corrupt_io_data(bdev_io,
error_disk->error_vector[bdev_io->type].corrupt_offset,
error_disk->error_vector[bdev_io->type].corrupt_value);
}
}
spdk_bdev_io_complete(bdev_io, status);
}
@ -187,6 +237,15 @@ vbdev_error_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bde
TAILQ_INSERT_TAIL(&error_disk->pending_ios, bdev_io, module_link);
error_disk->error_vector[bdev_io->type].error_num--;
break;
case VBDEV_IO_CORRUPT_DATA:
if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
error_disk->error_vector[bdev_io->type].error_num--;
vbdev_error_corrupt_io_data(bdev_io,
error_disk->error_vector[bdev_io->type].corrupt_offset,
error_disk->error_vector[bdev_io->type].corrupt_value);
}
/* fallthrough */
case 0:
rc = spdk_bdev_part_submit_request_ext(&ch->part_ch, bdev_io,
vbdev_error_complete_request);

3
module/bdev/error/vbdev_error.h
View File

@ -12,6 +12,7 @@
enum vbdev_error_type {
VBDEV_IO_FAILURE = 1,
VBDEV_IO_PENDING,
VBDEV_IO_CORRUPT_DATA,
};
typedef void (*spdk_delete_error_complete)(void *cb_arg, int bdeverrno);
@ -38,6 +39,8 @@ struct vbdev_error_inject_opts {
uint32_t io_type;
uint32_t error_type;
uint32_t error_num;
uint64_t corrupt_offset;
uint8_t corrupt_value;
};
/**

4
module/bdev/error/vbdev_error_rpc.c
View File

@ -45,6 +45,8 @@ rpc_error_bdev_decode_error_type(const struct spdk_json_val *val, void *out)
*error_type = VBDEV_IO_FAILURE;
} else if (spdk_json_strequal(val, "pending") == true) {
*error_type = VBDEV_IO_PENDING;
} else if (spdk_json_strequal(val, "corrupt_data") == true) {
*error_type = VBDEV_IO_CORRUPT_DATA;
} else {
SPDK_NOTICELOG("Invalid parameter value: error_type\n");
return -EINVAL;
@ -153,6 +155,8 @@ static const struct spdk_json_object_decoder rpc_error_information_decoders[] =
{"io_type", offsetof(struct rpc_error_information, opts.io_type), rpc_error_bdev_decode_io_type},
{"error_type", offsetof(struct rpc_error_information, opts.error_type), rpc_error_bdev_decode_error_type},
{"num", offsetof(struct rpc_error_information, opts.error_num), spdk_json_decode_uint32, true},
{"corrupt_offset", offsetof(struct rpc_error_information, opts.corrupt_offset), spdk_json_decode_uint64, true},
{"corrupt_value", offsetof(struct rpc_error_information, opts.corrupt_value), spdk_json_decode_uint8, true},
};
static void

11
python/spdk/rpc/bdev.py
View File

@ -1505,14 +1505,17 @@ def bdev_get_histogram(client, name):
return client.call('bdev_get_histogram', params)
def bdev_error_inject_error(client, name, io_type, error_type, num):
def bdev_error_inject_error(client, name, io_type, error_type, num,
corrupt_offset, corrupt_value):
"""Inject an error via an error bdev.
Args:
name: name of error bdev
io_type: one of "clear", "read", "write", "unmap", "flush", or "all"
error_type: one of "failure" or "pending"
error_type: one of "failure", "pending", or "corrupt_data"
num: number of commands to fail
corrupt_offset: offset in bytes to xor with corrupt_value
corrupt_value: value for xor (1-255, 0 is invalid)
"""
params = {
'name': name,
@ -1522,6 +1525,10 @@ def bdev_error_inject_error(client, name, io_type, error_type, num):
if num:
params['num'] = num
if corrupt_offset:
params['corrupt_offset'] = corrupt_offset
if corrupt_value:
params['corrupt_value'] = corrupt_value
return client.call('bdev_error_inject_error', params)

10
scripts/rpc.py
View File

@ -1165,14 +1165,20 @@ if __name__ == "__main__":
name=args.name,
io_type=args.io_type,
error_type=args.error_type,
num=args.num)
num=args.num,
corrupt_offset=args.corrupt_offset,
corrupt_value=args.corrupt_value)
p = subparsers.add_parser('bdev_error_inject_error', help='bdev inject error')
p.add_argument('name', help="""the name of the error injection bdev""")
p.add_argument('io_type', help="""io_type: 'clear' 'read' 'write' 'unmap' 'flush' 'all'""")
p.add_argument('error_type', help="""error_type: 'failure' 'pending'""")
p.add_argument('error_type', help="""error_type: 'failure' 'pending' 'corrupt_data'""")
p.add_argument(
'-n', '--num', help='the number of commands you want to fail', type=int)
p.add_argument(
'-o', '--corrupt-offset', help='the offset in bytes to xor with corrupt_value', type=int)
p.add_argument(
'-v', '--corrupt-value', help='the value for xor (1-255, 0 is invalid)', type=int)
p.set_defaults(func=bdev_error_inject_error)
def bdev_nvme_apply_firmware(args):