From efdd6edba20ecaf8ef9a476a70f15c2b5c4c8def Mon Sep 17 00:00:00 2001 From: Xiaodong Liu Date: Tue, 15 Jan 2019 20:52:11 +0800 Subject: [PATCH] nbd: optional nbd_device in start_nbd_disk Enhance RPC method start_nbd_disk to take nbd_device as one optional parameter. If it is not assigned, automaticly choose an available nbd device path from /dev/nbd0 to /dev/nbdN. For github issue #324: https://github.com/spdk/spdk/issues/324 Change-Id: I72c064d8bd476df342f5aa0af4d6120eb021c7ed Signed-off-by: Xiaodong Liu Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/440453 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Darek Stojaczyk --- CHANGELOG.md | 3 + lib/nbd/nbd_rpc.c | 144 ++++++++++++++++++++++++++++++++++++---- scripts/rpc.py | 2 +- scripts/rpc/nbd.py | 5 +- test/bdev/blockdev.sh | 1 + test/bdev/nbd_common.sh | 31 +++++++++ 6 files changed, 169 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b23080cf2..13a6c136b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,9 @@ that common usage of those functions is to call them just before terminating the New `get_spdk_version` RPC method is introduced to get version info of the running SPDK application. +The `start_nbd_disk` RPC method now take nbd_device as an optional parameter. If nbd_device +is specified, use that specified nbd device. If it's not specified, pick available one. + ## v19.01: ### ocf bdev diff --git a/lib/nbd/nbd_rpc.c b/lib/nbd/nbd_rpc.c index b31d30be1..05e5a2d2b 100644 --- a/lib/nbd/nbd_rpc.c +++ b/lib/nbd/nbd_rpc.c @@ -44,6 +44,10 @@ struct rpc_start_nbd_disk { char *bdev_name; char *nbd_device; + /* Used to search one available nbd device */ + int nbd_idx; + bool nbd_idx_specified; + struct spdk_jsonrpc_request *request; }; static void @@ -51,19 +55,107 @@ free_rpc_start_nbd_disk(struct rpc_start_nbd_disk *req) { free(req->bdev_name); free(req->nbd_device); + free(req); } static const struct spdk_json_object_decoder rpc_start_nbd_disk_decoders[] = { {"bdev_name", offsetof(struct rpc_start_nbd_disk, bdev_name), spdk_json_decode_string}, - {"nbd_device", offsetof(struct rpc_start_nbd_disk, nbd_device), spdk_json_decode_string}, + {"nbd_device", offsetof(struct rpc_start_nbd_disk, nbd_device), spdk_json_decode_string, true}, }; +/* Return 0 to indicate the nbd_device might be available, + * or non-zero to indicate the nbd_device is invalid or in using. + */ +static int +check_available_nbd_disk(char *nbd_device) +{ + char nbd_block_path[256]; + char tail[2]; + int rc; + unsigned int nbd_idx; + struct spdk_nbd_disk *nbd; + + /* nbd device path must be in format of /dev/nbd, with no tail. */ + rc = sscanf(nbd_device, "/dev/nbd%u%1s", &nbd_idx, tail); + if (rc != 1) { + return -1; + } + + /* make sure nbd_device is not registered inside SPDK */ + nbd = spdk_nbd_disk_find_by_nbd_path(nbd_device); + if (nbd) { + /* nbd_device is in using */ + return 1; + } + + /* A valid pid file in /sys/block indicates the device is in using */ + snprintf(nbd_block_path, 256, "/sys/block/nbd%u/pid", nbd_idx); + + rc = open(nbd_block_path, O_RDONLY); + if (rc < 0) { + if (errno == ENOENT) { + /* nbd_device might be available */ + return 0; + } else { + SPDK_ERRLOG("Failed to check PID file %s: %s\n", nbd_block_path, spdk_strerror(errno)); + return -1; + } + } + + close(rc); + + /* nbd_device is in using */ + return 1; +} + +static char * +find_available_nbd_disk(int nbd_idx, int *next_nbd_idx) +{ + int i, rc; + char nbd_device[20]; + + for (i = nbd_idx; ; i++) { + snprintf(nbd_device, 20, "/dev/nbd%d", i); + /* Check whether an nbd device exists in order to reach the last one nbd device */ + rc = access(nbd_device, F_OK); + if (rc != 0) { + break; + } + + rc = check_available_nbd_disk(nbd_device); + if (rc == 0) { + if (next_nbd_idx != NULL) { + *next_nbd_idx = i + 1; + } + + return strdup(nbd_device); + } + } + + return NULL; +} + static void spdk_rpc_start_nbd_done(void *cb_arg, struct spdk_nbd_disk *nbd, int rc) { - struct spdk_jsonrpc_request *request = cb_arg; + struct rpc_start_nbd_disk *req = cb_arg; + struct spdk_jsonrpc_request *request = req->request; struct spdk_json_write_ctx *w; + /* Check whether it's automatic nbd-device assignment */ + if (rc == -EBUSY && req->nbd_idx_specified == false) { + free(req->nbd_device); + + req->nbd_device = find_available_nbd_disk(req->nbd_idx, &req->nbd_idx); + if (req->nbd_device != NULL) { + spdk_nbd_start(req->bdev_name, req->nbd_device, + spdk_rpc_start_nbd_done, req); + return; + } + + SPDK_INFOLOG(SPDK_LOG_NBD, "There is no available nbd device.\n"); + } + if (rc) { spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); return; @@ -76,40 +168,64 @@ spdk_rpc_start_nbd_done(void *cb_arg, struct spdk_nbd_disk *nbd, int rc) spdk_json_write_string(w, spdk_nbd_get_path(nbd)); spdk_jsonrpc_end_result(request, w); + + free_rpc_start_nbd_disk(req); } static void spdk_rpc_start_nbd_disk(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { - struct rpc_start_nbd_disk req = {}; - struct spdk_nbd_disk *nbd; + struct rpc_start_nbd_disk *req; + int rc; + + req = calloc(1, sizeof(*req)); + if (req == NULL) { + SPDK_ERRLOG("could not allocate start_nbd_disk request.\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "Out of memory"); + return; + } if (spdk_json_decode_object(params, rpc_start_nbd_disk_decoders, SPDK_COUNTOF(rpc_start_nbd_disk_decoders), - &req)) { + req)) { SPDK_ERRLOG("spdk_json_decode_object failed\n"); goto invalid; } - if (req.nbd_device == NULL || req.bdev_name == NULL) { + if (req->bdev_name == NULL) { goto invalid; } - /* make sure nbd_device is not registered */ - nbd = spdk_nbd_disk_find_by_nbd_path(req.nbd_device); - if (nbd) { - goto invalid; + if (req->nbd_device != NULL) { + req->nbd_idx_specified = true; + rc = check_available_nbd_disk(req->nbd_device); + if (rc == 1) { + SPDK_DEBUGLOG(SPDK_LOG_NBD, "NBD device %s is in using.\n", req->nbd_device); + goto invalid; + } + + if (rc != 0) { + SPDK_DEBUGLOG(SPDK_LOG_NBD, "Illegal nbd_device %s.\n", req->nbd_device); + goto invalid; + } + } else { + req->nbd_idx = 0; + req->nbd_device = find_available_nbd_disk(req->nbd_idx, &req->nbd_idx); + if (req->nbd_device == NULL) { + SPDK_INFOLOG(SPDK_LOG_NBD, "There is no available nbd device.\n"); + goto invalid; + } } - spdk_nbd_start(req.bdev_name, req.nbd_device, - spdk_rpc_start_nbd_done, request); + req->request = request; + spdk_nbd_start(req->bdev_name, req->nbd_device, + spdk_rpc_start_nbd_done, req); - free_rpc_start_nbd_disk(&req); return; invalid: - free_rpc_start_nbd_disk(&req); + free_rpc_start_nbd_disk(req); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); } diff --git a/scripts/rpc.py b/scripts/rpc.py index 2f6661ba9..635d7099d 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1327,7 +1327,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p = subparsers.add_parser('start_nbd_disk', help='Export a bdev as a nbd disk') p.add_argument('bdev_name', help='Blockdev name to be exported. Example: Malloc0.') - p.add_argument('nbd_device', help='Nbd device name to be assigned. Example: /dev/nbd0.') + p.add_argument('nbd_device', help='Nbd device name to be assigned. Example: /dev/nbd0.', nargs='?') p.set_defaults(func=start_nbd_disk) def stop_nbd_disk(args): diff --git a/scripts/rpc/nbd.py b/scripts/rpc/nbd.py index 70cba1676..f0c294874 100644 --- a/scripts/rpc/nbd.py +++ b/scripts/rpc/nbd.py @@ -1,8 +1,9 @@ def start_nbd_disk(client, bdev_name, nbd_device): params = { - 'bdev_name': bdev_name, - 'nbd_device': nbd_device + 'bdev_name': bdev_name } + if nbd_device: + params['nbd_device'] = nbd_device return client.call('start_nbd_disk', params) diff --git a/test/bdev/blockdev.sh b/test/bdev/blockdev.sh index 1f457eff5..b8cb0b18e 100755 --- a/test/bdev/blockdev.sh +++ b/test/bdev/blockdev.sh @@ -41,6 +41,7 @@ function nbd_function_test() { echo "Process nbd pid: $nbd_pid" waitforlisten $nbd_pid $rpc_server + nbd_rpc_start_stop_verify $rpc_server "${bdev_list[*]}" nbd_rpc_data_verify $rpc_server "${bdev_list[*]}" "${nbd_list[*]}" $rpc_py -s $rpc_server delete_passthru_bdev TestPT diff --git a/test/bdev/nbd_common.sh b/test/bdev/nbd_common.sh index 2fa258b6c..4612a8f8e 100644 --- a/test/bdev/nbd_common.sh +++ b/test/bdev/nbd_common.sh @@ -13,6 +13,19 @@ function nbd_start_disks() { done } +function nbd_start_disks_without_nbd_idx() { + local rpc_server=$1 + local bdev_list=($2) + local i + local nbd_device + + for (( i=0; i<${#bdev_list[@]}; i++ )); do + nbd_device=$($rootdir/scripts/rpc.py -s $rpc_server start_nbd_disk ${bdev_list[$i]}) + # Wait for nbd device ready + waitfornbd $(basename ${nbd_device}) + done +} + function waitfornbd_exit() { local nbd_name=$1 @@ -90,3 +103,21 @@ function nbd_rpc_data_verify() { return 0 } + +function nbd_rpc_start_stop_verify() { + local rpc_server=$1 + local bdev_list=($2) + + nbd_start_disks_without_nbd_idx $rpc_server "${bdev_list[*]}" + + nbd_disks_json=`$rootdir/scripts/rpc.py -s $rpc_server get_nbd_disks` + nbd_disks_name=(`echo "${nbd_disks_json}" | jq -r '.[] | .nbd_device'`) + nbd_stop_disks $rpc_server "${nbd_disks_name[*]}" + + count=$(nbd_get_count $rpc_server) + if [ $count -ne 0 ]; then + return -1 + fi + + return 0 +}