blobcli: add script mode and readme

The readme explains how this mode works and includes a sample
test script that was used to test this mode.

Change-Id: I501b08004cc68157fe767ce8db61bf05f2eee391
Signed-off-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-on: https://review.gerrithub.io/380891
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

examples/blob/cli/README.md

@@ -0,0 +1,61 @@
+The blobcli tool has several options that are listed by using the -h command
+however the three operating modes are covered in more detail here:
+
+Command Mode
+------------
+This is the default and will just execute one command at a time. It's simple
+but the downside is that if you are going to interact quite a bit with the
+blobstore, the startup time for the application can be cumbersome.
+
+Shell Mode
+----------
+You startup shell mode by using the -S command. At that point you will get
+a "blob>" prompt where you can enter any of the commands, including -h,
+to execute them. You can stil enter just one at a time but the initial
+startup time for the application will not get in the way between commands
+anymore so it is much more usable.
+
+Script (aka test) Mode
+----------------------
+In script mode you just supply one command with a filename when you start
+the cli, for example `blobcli -T test.bs` will feed the tool the file
+called test.bs which contains a series of commands that will all run
+automatically and, like shell mode, will only initialize one time so is
+quick.
+
+The script file format (example) is shown below.  Comments are allowed and
+each line should contain one valid command (and its parameters) only. In
+order to operate on blobs via their ID value, use the token $Bn where n
+represents the instance of the blob created in the script.
+
+For example, the line `-s $B0` will operate on the blobid of the first
+blob created in the script (0 index based). `$B2` represents the third
+blob created in the script.
+
+If you start test mode with the additional "ignore" option, any invalid
+script lines will simply be skipped, otherwise the tool will exit if
+it runs into an invalid line (ie './blobcli -T test.bs ignore`).
+
+Sample test/bs file:
+~~{.sh}
+# this is a comment
+-i
+-s bs
+-l bdevs
+-n 1
+-s bs
+-s $B0
+-n 2
+-s $B1
+-m $B0 Makefile
+-d $B0 M.blob
+-f $B1 65
+-d $B1 65.blob
+-s bs
+-x $B0 b0key boval
+-x $B1 b1key b1val
+-r $B0 b0key
+-s $B0
+-s $B1
+-s bs
+~~

examples/blob/cli/blobcli.c

@@ -59,11 +59,13 @@ static const char *bdev_name = "Nvme0n1";
 /*
  * CMD mode runs one command at a time which can be annoying as the init takes
  * a few seconds, so the shell mode, invoked with -S, does the init once and gives
- * the user an interactive shell instead.
+ * the user an interactive shell instead. With script mode init is also done just
+ * once.
  */
 enum cli_mode_type {
 	CLI_MODE_CMD,
-	CLI_MODE_SHELL
+	CLI_MODE_SHELL,
+	CLI_MODE_SCRIPT
 };
 
 enum cli_action_type {
@@ -81,14 +83,15 @@ enum cli_action_type {
 	CLI_LIST_BLOBS,
 	CLI_INIT_BS,
 	CLI_SHELL_EXIT,
-	CLI_HELP
+	CLI_HELP,
 };
 
 #define BUFSIZE 255
-#define MAX_ARGS 6
+#define MAX_ARGS 16
 #define ALIGN_4K 4096
 #define STARTING_PAGE 0
 #define NUM_PAGES 1
+
 /*
  * The CLI uses the SPDK app framework so is async and callback driven. A
  * pointer to this structure is passed to SPDK calls and returned in the
@@ -121,17 +124,28 @@ struct cli_context_t {
 	int argc;
 	char *argv[MAX_ARGS];
 	bool app_started;
+	char script_file[BUFSIZE + 1];
 };
 
+/* we store a bunch of stuff in a global struct for use by scripting mode */
+#define MAX_SCRIPT_LINES 64
+#define MAX_SCRIPT_BLOBS 16
+struct cli_script_t {
+	spdk_blob_id blobid[MAX_SCRIPT_BLOBS];
+	int blobid_idx;
+	int max_index;
+	int cmdline_idx;
+	bool ignore_errors;
+	char *cmdline[MAX_SCRIPT_LINES];
+};
+struct cli_script_t g_script;
+
 /*
  * Common printing of commands for CLI and shell modes.
  */
 static void
-print_cmds(char *msg)
+print_cmds(void)
 {
-	if (msg) {
-		printf("%s", msg);
-	}
 	printf("\nCommands include:\n");
 	printf("\t-d <blobid> filename - dump contents of a blob to a file\n");
 	printf("\t-f <blobid> value - fill a blob with a decimal value\n");
@@ -146,6 +160,7 @@ print_cmds(char *msg)
 	printf("\t-x <blobid> name value - set xattr name/value pair\n");
 	printf("\t-X - exit when in interactive shell mode\n");
 	printf("\t-S - enter interactive shell mode\n");
+	printf("\t-T <filename> - automated script mode\n");
 	printf("\n");
 }
 
@@ -153,18 +168,23 @@ print_cmds(char *msg)
  * Prints usage and relevant error message.
  */
 static void
-usage(char *msg)
+usage(struct cli_context_t *cli_context, char *msg)
 {
 	if (msg) {
 		printf("%s", msg);
 	}
+
+	if (cli_context && cli_context->cli_mode == CLI_MODE_CMD) {
 		printf("Version %s\n", SPDK_VERSION_STRING);
 		printf("Usage: %s [-c SPDK config_file] Command\n", program_name);
 		printf("\n%s is a command line tool for interacting with blobstore\n",
 		       program_name);
 		printf("on the underlying device specified in the conf file passed\n");
 		printf("in as a command line option.\n");
-	print_cmds("");
+	}
+	if (cli_context && cli_context->cli_mode != CLI_MODE_SCRIPT) {
+		print_cmds();
+	}
 }
 
 /*
@@ -176,6 +196,14 @@ cli_cleanup(struct cli_context_t *cli_context)
 	if (cli_context->buff) {
 		spdk_dma_free(cli_context->buff);
 	}
+	if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+		int i;
+
+		for (i = 0; i <= g_script.max_index; i++) {
+			free(g_script.cmdline[i]);
+			g_script.cmdline[i] = NULL;
+		}
+	}
 	free(cli_context);
 }
 
@@ -315,6 +343,11 @@ blob_create_cb(void *arg1, spdk_blob_id blobid, int bserrno)
 	cli_context->blobid = blobid;
 	printf("New blob id %" PRIu64 "\n", cli_context->blobid);
 
+	/* if we're in script mode, we need info on all blobids for later */
+	if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+		g_script.blobid[g_script.blobid_idx++] = blobid;
+	}
+
 	/* We have to open the blob before we can do things like resize. */
 	spdk_bs_md_open_blob(cli_context->bs, cli_context->blobid,
 			     open_now_resize_cb, cli_context);
@@ -713,7 +746,6 @@ write_cb(void *arg1, int bserrno)
 		spdk_bs_md_close_blob(&cli_context->blob, close_cb,
 				      cli_context);
 	}
-
 }
 
 /*
@@ -743,7 +775,7 @@ fill_blob_cb(void *arg1, struct spdk_blob *blob, int bserrno)
 
 	memset(cli_context->buff, cli_context->fill_value,
 	       cli_context->page_size);
-	printf("\n");
+	printf("Working");
 	spdk_bs_io_write_blob(cli_context->blob, cli_context->channel,
 			      cli_context->buff,
 			      STARTING_PAGE, NUM_PAGES, write_cb, cli_context);
@@ -927,47 +959,56 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 	int cmd_chosen = 0;
 	char resp;
 
-	while ((op = getopt(argc, argv, "c:d:f:hil:m:n:p:r:s:SXx:")) != -1) {
+	while ((op = getopt(argc, argv, "c:d:f:hil:m:n:p:r:s:ST:Xx:")) != -1) {
 		switch (op) {
 		case 'c':
 			if (cli_context->app_started == false) {
 				cmd_chosen++;
 				cli_context->config_file = optarg;
 			} else {
-				print_cmds("ERROR: -c option not valid during shell mode.\n");
+				usage(cli_context, "ERROR: -c option not valid during shell mode.\n");
 			}
 			break;
 		case 'd':
 			cmd_chosen++;
 			cli_context->action = CLI_DUMP;
 			cli_context->blobid = atoll(optarg);
+			snprintf(cli_context->file, BUFSIZE, "%s", argv[optind]);
 			break;
 		case 'f':
 			cmd_chosen++;
 			cli_context->action = CLI_FILL;
 			cli_context->blobid = atoll(optarg);
+			cli_context->fill_value = atoi(argv[optind]);
 			break;
 		case 'h':
 			cmd_chosen++;
 			cli_context->action = CLI_HELP;
 			break;
 		case 'i':
-			printf("You entire blobstore will be destroyed. Are you sure? (y/n) ");
+			if (cli_context->cli_mode != CLI_MODE_SCRIPT) {
+				printf("Your entire blobstore will be destroyed. Are you sure? (y/n) ");
 				if (scanf("%c%*c", &resp)) {
 					if (resp == 'y' || resp == 'Y') {
 						cmd_chosen++;
 						cli_context->action = CLI_INIT_BS;
 					} else {
 						if (cli_context->cli_mode == CLI_MODE_CMD) {
-						exit(0);
+							spdk_app_stop(0);
+							return false;
 						}
 					}
 				}
+			} else {
+				cmd_chosen++;
+				cli_context->action = CLI_INIT_BS;
+			}
 			break;
 		case 'r':
 			cmd_chosen++;
 			cli_context->action = CLI_REM_XATTR;
 			cli_context->blobid = atoll(optarg);
+			snprintf(cli_context->key, BUFSIZE, "%s", argv[optind]);
 			break;
 		case 'l':
 			if (strcmp("bdevs", optarg) == 0) {
@@ -977,18 +1018,14 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 				cmd_chosen++;
 				cli_context->action = CLI_LIST_BLOBS;
 			} else {
-				if (cli_context->cli_mode == CLI_MODE_CMD) {
+				usage(cli_context, "ERROR: invalid option for list\n");
-					usage("ERROR: invalid option for list\n");
-					exit(-1);
-				} else {
-					print_cmds("ERROR: invalid option for list\n");
-				}
 			}
 			break;
 		case 'm':
 			cmd_chosen++;
 			cli_context->action = CLI_IMPORT;
 			cli_context->blobid = atoll(optarg);
+			snprintf(cli_context->file, BUFSIZE, "%s", argv[optind]);
 			break;
 		case 'n':
 			cmd_chosen++;
@@ -996,12 +1033,7 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 			if (cli_context->num_clusters > 0) {
 				cli_context->action = CLI_CREATE_BLOB;
 			} else {
-				if (cli_context->cli_mode == CLI_MODE_CMD) {
+				usage(cli_context, "ERROR: invalid option for new\n");
-					usage("ERROR: invalid option for new\n");
-					exit(-1);
-				} else {
-					print_cmds("ERROR: invalid option for new\n");
-				}
 			}
 			break;
 		case 'p':
@@ -1025,6 +1057,20 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 				cli_context->blobid = atoll(optarg);
 			}
 			break;
+		case 'T':
+			if (cli_context->cli_mode == CLI_MODE_CMD) {
+				cmd_chosen++;
+				cli_context->cli_mode = CLI_MODE_SCRIPT;
+				if (argv[optind] && (strcmp("ignore", argv[optind]) == 0)) {
+					g_script.ignore_errors = true;
+				} else {
+					g_script.ignore_errors = false;
+				}
+				snprintf(cli_context->script_file, BUFSIZE, "%s", optarg);
+			} else {
+				cli_context->action = CLI_NONE;
+			}
+			break;
 		case 'X':
 			cmd_chosen++;
 			cli_context->action = CLI_SHELL_EXIT;
@@ -1033,31 +1079,22 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 			cmd_chosen++;
 			cli_context->action = CLI_SET_XATTR;
 			cli_context->blobid = atoll(optarg);
+			snprintf(cli_context->key, BUFSIZE, "%s", argv[optind]);
+			snprintf(cli_context->value, BUFSIZE, "%s", argv[optind + 1]);
 			break;
 		default:
-			if (cli_context->cli_mode == CLI_MODE_CMD) {
+			usage(cli_context, "ERROR: invalid option\n");
-				usage("ERROR: invalid option\n");
-				exit(-1);
-			} else {
-				print_cmds("ERROR: invalid option\n");
-			}
 		}
 		/* config file is the only option that can be combined */
 		if (op != 'c') {
 			if (cmd_chosen > 1) {
-				if (cli_context->cli_mode == CLI_MODE_CMD) {
+				usage(cli_context, "Error: Please choose only one command\n");
-					usage("Error: Please choose only one command\n");
-					cli_cleanup(cli_context);
-					exit(1);
-				} else {
-					print_cmds("Error: Please choose only one command\n");
-				}
 			}
 		}
 	}
 
 	if (cli_context->cli_mode == CLI_MODE_CMD && cmd_chosen == 0) {
-		usage("Error: Please choose a command.\n");
+		usage(cli_context, "Error: Please choose a command.\n");
 		exit(1);
 	}
 
@@ -1084,6 +1121,137 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 	return (cmd_chosen > 0);
 }
 
+/*
+ * In script mode, we parsed a script file at startup and saved off a bunch of cmd
+ * lines that we now parse with each run of cli_start so we us the same cmd parser
+ * as cmd and shell modes.
+ */
+static bool
+line_parser(struct cli_context_t *cli_context)
+{
+	bool cmd_chosen;
+	char *tok = NULL;
+	int blob_num = 0;
+	int start_idx = cli_context->argc;
+	int i;
+
+	printf("\nSCRIPT NOW PROCESSING: %s\n", g_script.cmdline[g_script.cmdline_idx]);
+	tok = strtok(g_script.cmdline[g_script.cmdline_idx], " ");
+	while (tok != NULL) {
+		/*
+		 * We support one replaceable token right now, a $Bn
+		 * represents the blobid that was created in position n
+		 * so fish this out now and use it here.
+		 */
+		cli_context->argv[cli_context->argc] = strdup(tok);
+		if (tok[0] == '$' && tok[1] == 'B') {
+			tok += 2;
+			blob_num = atoi(tok);
+			cli_context->argv[cli_context->argc] =
+				realloc(cli_context->argv[cli_context->argc], BUFSIZE);
+			if (cli_context->argv[cli_context->argc] == NULL) {
+				printf("ERROR: unable to realloc memory\n");
+				spdk_app_stop(-1);
+			}
+			if (g_script.blobid[blob_num] == 0) {
+				printf("ERROR: There is no blob for $B%d\n",
+				       blob_num);
+			}
+			snprintf(cli_context->argv[cli_context->argc], BUFSIZE,
+				 "%" PRIu64, g_script.blobid[blob_num]);
+		}
+		cli_context->argc++;
+		tok = strtok(NULL, " ");
+	}
+
+	/* call parse cmd line with user input as args */
+	cmd_chosen = cmd_parser(cli_context->argc, &cli_context->argv[0], cli_context);
+
+	/* free strdup memory and reset arg count for next shell interaction */
+	for (i = start_idx; i < cli_context->argc; i++) {
+		free(cli_context->argv[i]);
+		cli_context->argv[i] = NULL;
+	}
+	cli_context->argc = 1;
+
+	g_script.cmdline_idx++;
+	assert(g_script.cmdline_idx < MAX_SCRIPT_LINES);
+
+	if (cmd_chosen == false) {
+		printf("ERROR: Invalid script line starting with: %s\n\n",
+		       g_script.cmdline[g_script.cmdline_idx - 1]);
+		if (g_script.ignore_errors == false) {
+			printf("** Aborting **\n");
+			cli_context->action = CLI_SHELL_EXIT;
+			cmd_chosen = true;
+			unload_bs(cli_context, "", 0);
+		} else {
+			printf("** Skipping **\n");
+		}
+	}
+
+	return cmd_chosen;
+}
+
+/*
+ * For script mode, we read a series of commands from a text file and store them
+ * in a global struct. That, along with the cli_mode that tells us we're in
+ * script mode is what feeds the rest of the app in the same way as is it were
+ * getting commands from shell mode.
+ */
+static void
+parse_script(struct cli_context_t *cli_context)
+{
+	FILE *fp = NULL;
+	size_t bufsize = BUFSIZE;
+	int64_t bytes_in = 0;
+	int i = 0;
+
+	/* initialize global script values */
+	for (i = 0; i < MAX_SCRIPT_BLOBS; i++) {
+		g_script.blobid[i] = 0;
+	}
+	g_script.blobid_idx = 0;
+	g_script.cmdline_idx = 0;
+	i = 0;
+
+	fp = fopen(cli_context->script_file, "r");
+	if (fp == NULL) {
+		printf("ERROR: unable to open script: %s\n",
+		       cli_context->script_file);
+		cli_cleanup(cli_context);
+		exit(-1);
+	}
+
+	do {
+		bytes_in = getline(&g_script.cmdline[i], &bufsize, fp);
+		if (bytes_in > 0) {
+			/* replace newline with null */
+			spdk_str_chomp(g_script.cmdline[i]);
+
+			/* ignore comments */
+			if (g_script.cmdline[i][0] != '#') {
+				i++;
+			}
+		}
+	} while (bytes_in != -1 && i < MAX_SCRIPT_LINES);
+	fclose(fp);
+
+	/* add an exit cmd in case they didn't */
+	g_script.cmdline[i] = realloc(g_script.cmdline[i], BUFSIZE);
+	if (g_script.cmdline[i] == NULL)  {
+		int j;
+
+		for (j = 0; j < i; j++) {
+			free(g_script.cmdline[j]);
+			g_script.cmdline[j] = NULL;
+		}
+		unload_bs(cli_context, "ERROR: unable to alloc memory.\n", 0);
+	}
+	snprintf(g_script.cmdline[i], BUFSIZE, "%s", "-X");
+	g_script.max_index = i;
+}
+
 /*
  * Provides for a shell interface as opposed to one shot command line.
  */
@@ -1142,6 +1310,14 @@ cli_start(void *arg1, void *arg2)
 {
 	struct cli_context_t *cli_context = arg1;
 
+	/*
+	 * If we're in script mode, we already have a list of commands so
+	 * just need to pull them out one at a time and process them.
+	 */
+	if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+		while (line_parser(cli_context) == false);
+	}
+
 	/*
 	 * The initial cmd line options are parsed once before this function is
 	 * called so if there is no action, we're in shell mode and will loop
@@ -1181,7 +1357,7 @@ cli_start(void *arg1, void *arg2)
 		spdk_app_stop(0);
 		break;
 	case CLI_HELP:
-		print_cmds("");
+		usage(cli_context, "");
 		unload_complete(cli_context, 0);
 		break;
 	default:
@@ -1196,11 +1372,12 @@ main(int argc, char **argv)
 {
 	struct spdk_app_opts opts = {};
 	struct cli_context_t *cli_context = NULL;
+	bool cmd_chosen;
 	int rc = 0;
 
 	if (argc < 2) {
-		usage("ERROR: Invalid option\n");
+		usage(cli_context, "ERROR: Invalid option\n");
-		exit(1);
+		exit(-1);
 	}
 
 	cli_context = calloc(1, sizeof(struct cli_context_t));
@@ -1216,12 +1393,17 @@ main(int argc, char **argv)
 	cli_context->argc = 1;
 
 	/* parse command line */
-	cmd_parser(argc, argv, cli_context);
+	cmd_chosen = cmd_parser(argc, argv, cli_context);
 	free(cli_context->argv[0]);
+	cli_context->argv[0] = NULL;
+	if (cmd_chosen == false) {
+		cli_cleanup(cli_context);
+		exit(-1);
+	}
 
 	/* after displaying help, just exit */
 	if (cli_context->action == CLI_HELP) {
-		usage("");
+		usage(cli_context, "");
 		cli_cleanup(cli_context);
 		exit(-1);
 	}
@@ -1240,6 +1422,19 @@ main(int argc, char **argv)
 		exit(1);
 	}
 
+	/*
+	 * For script mode we keep a bunch of stuff in a global since
+	 * none if it is passed back and forth to SPDK.
+	 */
+	if (cli_context->cli_mode == CLI_MODE_SCRIPT) {
+		/*
+		 * Now we'll build up the global which will direct this run of the app
+		 * as it will have a list (g_script) of all of the commands line by
+		 * line as if they were typed in on the shell at cmd line.
+		 */
+		parse_script(cli_context);
+	}
+
 	/* Set default values in opts struct along with name and conf file. */
 	spdk_app_opts_init(&opts);
 	opts.name = "blobcli";