bdev/error: Change the global error information to per bdev.

Change-Id: Iea492a36d23caa5d514f2c80e168486f7f616fda
Signed-off-by: cunyinch <cunyin.chang@intel.com>
Reviewed-on: https://review.gerrithub.io/363676
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/bdev/error/vbdev_error.c

@@ -51,22 +51,39 @@
 struct vbdev_error_disk {
 	struct spdk_bdev		disk;
 	struct spdk_bdev		*base_bdev;
+	uint32_t			io_type_mask;
+	uint32_t			error_num;
 	TAILQ_ENTRY(vbdev_error_disk)	tailq;
 };
 
-static uint32_t g_io_type_mask;
-static uint32_t g_error_num;
 static pthread_mutex_t g_vbdev_error_mutex = PTHREAD_MUTEX_INITIALIZER;
 static TAILQ_HEAD(, vbdev_error_disk) g_vbdev_error_disks = TAILQ_HEAD_INITIALIZER(
 			g_vbdev_error_disks);
 
-void
+int
-spdk_vbdev_inject_error(uint32_t io_type_mask, uint32_t error_num)
+spdk_vbdev_inject_error(char *name, uint32_t io_type_mask, uint32_t error_num)
 {
+	struct spdk_bdev *bdev;
+	struct vbdev_error_disk *error_disk;
+
 	pthread_mutex_lock(&g_vbdev_error_mutex);
-	g_io_type_mask = io_type_mask;
+	bdev = spdk_bdev_get_by_name(name);
-	g_error_num = error_num;
+	if (!bdev) {
+		SPDK_ERRLOG("Could not find ErrorInjection bdev %s\n", name);
+		pthread_mutex_unlock(&g_vbdev_error_mutex);
+		return -1;
+	}
+
+	TAILQ_FOREACH(error_disk, &g_vbdev_error_disks, tailq) {
+		if (bdev == &error_disk->disk) {
+			break;
+		}
+	}
+
+	error_disk->io_type_mask = io_type_mask;
+	error_disk->error_num = error_num;
 	pthread_mutex_unlock(&g_vbdev_error_mutex);
+	return 0;
 }
 
 static void
@@ -105,20 +122,13 @@ vbdev_error_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev
 
 	io_type_mask = 1U << bdev_io->type;
 
-	if (g_error_num == 0 || !(g_io_type_mask & io_type_mask)) {
+	if (error_disk->error_num == 0 || !(error_disk->io_type_mask & io_type_mask)) {
 		spdk_bdev_io_resubmit(bdev_io, error_disk->base_bdev);
 		return;
 	}
 
-	pthread_mutex_lock(&g_vbdev_error_mutex);
+	error_disk->error_num--;
-	/* check again to make sure g_error_num has not been decremented since we checked it above */
+	spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
-	if (g_error_num == 0) {
-		spdk_bdev_io_resubmit(bdev_io, error_disk->base_bdev);
-	} else {
-		g_error_num--;
-		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
-	}
-	pthread_mutex_unlock(&g_vbdev_error_mutex);
 }
 
 static void

lib/bdev/error/vbdev_error.h

@@ -38,6 +38,6 @@
 #include "spdk/bdev.h"
 
 int spdk_vbdev_error_create(struct spdk_bdev *base_bdev);
-void spdk_vbdev_inject_error(uint32_t io_type_mask, uint32_t error_num);
+int spdk_vbdev_inject_error(char *name, uint32_t io_type_mask, uint32_t error_num);
 
 #endif // SPDK_BLOCKDEV_ERROR_H

lib/bdev/error/vbdev_error_rpc.c

@@ -123,11 +123,13 @@ invalid:
 SPDK_RPC_REGISTER("construct_error_bdev", spdk_rpc_construct_error_bdev)
 
 struct rpc_error_information {
+	char *name;
 	char *type;
 	uint32_t num;
 };
 
 static const struct spdk_json_object_decoder rpc_error_information_decoders[] = {
+	{"name", offsetof(struct rpc_error_information, name), spdk_json_decode_string},
 	{"type", offsetof(struct rpc_error_information, type), spdk_json_decode_string},
 	{"num", offsetof(struct rpc_error_information, num), spdk_json_decode_uint32, true},
 };
@@ -135,6 +137,7 @@ static const struct spdk_json_object_decoder rpc_error_information_decoders[] =
 static void
 free_rpc_error_information(struct rpc_error_information *p)
 {
+	free(p->name);
 	free(p->type);
 }
 
@@ -145,7 +148,8 @@ spdk_rpc_bdev_inject_error(struct spdk_jsonrpc_server_conn *conn,
 {
 	struct rpc_error_information req = {};
 	struct spdk_json_write_ctx *w;
-	uint32_t ret;
+	uint32_t io_type;
+	int ret;
 
 	if (spdk_json_decode_object(params, rpc_error_information_decoders,
 				    SPDK_COUNTOF(rpc_error_information_decoders),
@@ -154,12 +158,16 @@ spdk_rpc_bdev_inject_error(struct spdk_jsonrpc_server_conn *conn,
 		goto invalid;
 	}
 
-	ret = spdk_rpc_error_bdev_io_type_parse(req.type);
+	io_type = spdk_rpc_error_bdev_io_type_parse(req.type);
-	if (ret == ERROR_BDEV_IO_TYPE_INVALID) {
+	if (io_type == ERROR_BDEV_IO_TYPE_INVALID) {
+		goto invalid;
+	}
+
+	ret = spdk_vbdev_inject_error(req.name, io_type, req.num);
+	if (ret) {
 		goto invalid;
 	}
 
-	spdk_vbdev_inject_error(ret, req.num);
 	free_rpc_error_information(&req);
 
 	if (id == NULL) {

scripts/rpc.py

@@ -455,6 +455,7 @@ p.set_defaults(func=delete_nvmf_subsystem)
 
 def bdev_inject_error(args):
     params = {
+        'name': args.name,
         'type': args.type,
         'num': args.num,
     }
@@ -462,6 +463,7 @@ def bdev_inject_error(args):
     jsonrpc_call('bdev_inject_error', params)
 
 p = subparsers.add_parser('bdev_inject_error', help='bdev inject error')
+p.add_argument('name', help="""the name of the error injection bdev""")
 p.add_argument('type', help="""type: 'clear' 'read' 'write' 'unmap' 'flush' 'reset' 'all'""")
 p.add_argument('-n', '--num', help='the number of commands you want to fail', type=int, default=1)
 p.set_defaults(func=bdev_inject_error)

test/iscsi_tgt/ext4test/ext4test.sh

@@ -54,7 +54,7 @@ trap 'for new_dir in `dir -d /mnt/*dir`; do umount $new_dir; rm -rf $new_dir; do
 sleep 1
 
 echo "Test error injection"
-$rpc_py bdev_inject_error 'all' -n 1000
+$rpc_py bdev_inject_error EE_Malloc0 'all' -n 1000
 
 dev=$(iscsiadm -m session -P 3 | grep "Attached scsi disk" | awk '{print $4}')
 
@@ -71,7 +71,7 @@ else
 fi
 set -e
 
-$rpc_py bdev_inject_error 'clear'
+$rpc_py bdev_inject_error EE_Malloc0 'clear'
 echo "Error injection test done"
 
 iscsicleanup