diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index f338014a9..c07f6551e 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -1160,9 +1160,21 @@ This method is available only if SPDK was build with Ceph RBD support. Name | Optional | Type | Description ----------------------- | -------- | ----------- | ----------- name | Optional | string | Bdev name +user_id | Optional | string | Ceph ID (i.e. admin, not client.admin) pool_name | Required | string | Pool name rbd_name | Required | string | Image name block_size | Required | number | Block size +config | Optional | string map | Explicit librados configuration + +If no config is specified, Ceph configuration files must exist with +all relevant settings for accessing the pool. If a config map is +passed, the configuration files are ignored and instead all key/value +pairs are passed to rados_conf_set to configure cluster access. In +practice, "mon_host" (= list of monitor address+port) and "key" (= the +secret key stored in Ceph keyrings) are enough. + +When accessing the image as some user other than "admin" (the +default), the "user_id" has to be set. ### Result @@ -1170,13 +1182,17 @@ Name of newly created bdev. ### Example -Example request: +Example request with `key` from `/etc/ceph/ceph.client.admin.keyring`: ~~~ { "params": { "pool_name": "rbd", "rbd_name": "foo", + "config": { + "mon_host": "192.168.7.1:6789,192.168.7.2:6789", + "key": "AQDwf8db7zR1GRAA5k7NKXjS5S5V4mntwUDnGQ==", + } "block_size": 4096 }, "jsonrpc": "2.0", diff --git a/lib/bdev/rbd/bdev_rbd.c b/lib/bdev/rbd/bdev_rbd.c index 34c2466b3..593ecf2ea 100644 --- a/lib/bdev/rbd/bdev_rbd.c +++ b/lib/bdev/rbd/bdev_rbd.c @@ -59,7 +59,9 @@ static int bdev_rbd_count = 0; struct bdev_rbd { struct spdk_bdev disk; char *rbd_name; + char *user_id; char *pool_name; + char **config; rbd_image_info_t info; TAILQ_ENTRY(bdev_rbd) tailq; struct spdk_poller *reset_timer; @@ -90,27 +92,78 @@ bdev_rbd_free(struct bdev_rbd *rbd) free(rbd->disk.name); free(rbd->rbd_name); + free(rbd->user_id); free(rbd->pool_name); + spdk_bdev_rbd_free_config(rbd->config); free(rbd); } +void +spdk_bdev_rbd_free_config(char **config) +{ + char **entry; + + if (config) { + for (entry = config; *entry; entry++) { + free(*entry); + } + free(config); + } +} + +char ** +spdk_bdev_rbd_dup_config(const char *const *config) +{ + size_t count; + char **copy; + + if (!config) { + return NULL; + } + for (count = 0; config[count]; count++) {} + copy = calloc(count + 1, sizeof(*copy)); + if (!copy) { + return NULL; + } + for (count = 0; config[count]; count++) { + if (!(copy[count] = strdup(config[count]))) { + spdk_bdev_rbd_free_config(copy); + return NULL; + } + } + return copy; +} + static int -bdev_rados_context_init(const char *rbd_pool_name, rados_t *cluster, - rados_ioctx_t *io_ctx) +bdev_rados_context_init(const char *user_id, const char *rbd_pool_name, const char *const *config, + rados_t *cluster, rados_ioctx_t *io_ctx) { int ret; - ret = rados_create(cluster, NULL); + ret = rados_create(cluster, user_id); if (ret < 0) { SPDK_ERRLOG("Failed to create rados_t struct\n"); return -1; } - ret = rados_conf_read_file(*cluster, NULL); - if (ret < 0) { - SPDK_ERRLOG("Failed to read conf file\n"); - rados_shutdown(*cluster); - return -1; + if (config) { + const char *const *entry = config; + while (*entry) { + ret = rados_conf_set(*cluster, entry[0], entry[1]); + if (ret < 0) { + SPDK_ERRLOG("Failed to set %s = %s\n", entry[0], entry[1]); + rados_shutdown(*cluster); + return -1; + } + entry += 2; + } + } else { + ret = rados_conf_read_file(*cluster, NULL); + if (ret < 0) { + SPDK_ERRLOG("Failed to read conf file\n"); + rados_shutdown(*cluster); + return -1; + } } ret = rados_connect(*cluster); @@ -132,17 +185,18 @@ bdev_rados_context_init(const char *rbd_pool_name, rados_t *cluster, } static int -bdev_rbd_init(const char *rbd_pool_name, const char *rbd_name, rbd_image_info_t *info) +bdev_rbd_init(const char *user_id, const char *rbd_pool_name, const char *const *config, + const char *rbd_name, rbd_image_info_t *info) { int ret; rados_t cluster = NULL; rados_ioctx_t io_ctx = NULL; rbd_image_t image = NULL; - ret = bdev_rados_context_init(rbd_pool_name, &cluster, &io_ctx); + ret = bdev_rados_context_init(user_id, rbd_pool_name, config, &cluster, &io_ctx); if (ret < 0) { - SPDK_ERRLOG("Failed to create rados context for rbd_pool=%s\n", - rbd_pool_name); + SPDK_ERRLOG("Failed to create rados context for user_id=%s and rbd_pool=%s\n", + user_id ? user_id : "admin (the default)", rbd_pool_name); return -1; } @@ -493,10 +547,12 @@ bdev_rbd_create_cb(void *io_device, void *ctx_buf) ch->io_ctx = NULL; ch->pfd.fd = -1; - ret = bdev_rados_context_init(ch->disk->pool_name, &ch->cluster, &ch->io_ctx); + ret = bdev_rados_context_init(ch->disk->user_id, ch->disk->pool_name, + (const char *const *)ch->disk->config, + &ch->cluster, &ch->io_ctx); if (ret < 0) { - SPDK_ERRLOG("Failed to create rados context for rbd_pool=%s\n", - ch->disk->pool_name); + SPDK_ERRLOG("Failed to create rados context for user_id %s and rbd_pool=%s\n", + ch->disk->user_id ? ch->disk->user_id : "admin (the default)", ch->disk->pool_name); goto err; } @@ -558,6 +614,22 @@ bdev_rbd_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) spdk_json_write_name(w, "rbd_name"); spdk_json_write_string(w, rbd_bdev->rbd_name); + if (rbd_bdev->user_id) { + spdk_json_write_named_string(w, "user_id", rbd_bdev->user_id); + } + + if (rbd_bdev->config) { + char **entry = rbd_bdev->config; + + spdk_json_write_name(w, "config"); + spdk_json_write_object_begin(w); + while (*entry) { + spdk_json_write_named_string(w, entry[0], entry[1]); + entry += 2; + } + spdk_json_write_object_end(w); + } + spdk_json_write_object_end(w); return 0; @@ -577,6 +649,22 @@ bdev_rbd_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w spdk_json_write_named_string(w, "pool_name", rbd->pool_name); spdk_json_write_named_string(w, "rbd_name", rbd->rbd_name); spdk_json_write_named_uint32(w, "block_size", bdev->blocklen); + if (rbd->user_id) { + spdk_json_write_named_string(w, "user_id", rbd->user_id); + } + + if (rbd->config) { + char **entry = rbd->config; + + spdk_json_write_name(w, "config"); + spdk_json_write_object_begin(w); + while (*entry) { + spdk_json_write_named_string(w, entry[0], entry[1]); + entry += 2; + } + spdk_json_write_object_end(w); + } + spdk_json_write_object_end(w); spdk_json_write_object_end(w); @@ -592,7 +680,9 @@ static const struct spdk_bdev_fn_table rbd_fn_table = { }; struct spdk_bdev * -spdk_bdev_rbd_create(const char *name, const char *pool_name, const char *rbd_name, +spdk_bdev_rbd_create(const char *name, const char *user_id, const char *pool_name, + const char *const *config, + const char *rbd_name, uint32_t block_size) { struct bdev_rbd *rbd; @@ -614,13 +704,28 @@ spdk_bdev_rbd_create(const char *name, const char *pool_name, const char *rbd_na return NULL; } + if (user_id) { + rbd->user_id = strdup(user_id); + if (!rbd->user_id) { + bdev_rbd_free(rbd); + return NULL; + } + } + rbd->pool_name = strdup(pool_name); if (!rbd->pool_name) { bdev_rbd_free(rbd); return NULL; } - ret = bdev_rbd_init(rbd->pool_name, rbd_name, &rbd->info); + if (config && !(rbd->config = spdk_bdev_rbd_dup_config(config))) { + bdev_rbd_free(rbd); + return NULL; + } + + ret = bdev_rbd_init(rbd->user_id, rbd->pool_name, + (const char *const *)rbd->config, + rbd_name, &rbd->info); if (ret < 0) { bdev_rbd_free(rbd); SPDK_ERRLOG("Failed to init rbd device\n"); @@ -727,7 +832,8 @@ bdev_rbd_library_init(void) } } - if (spdk_bdev_rbd_create(NULL, pool_name, rbd_name, block_size) == NULL) { + /* TODO(?): user_id and rbd config values */ + if (spdk_bdev_rbd_create(NULL, NULL, pool_name, NULL, rbd_name, block_size) == NULL) { rc = -1; goto end; } diff --git a/lib/bdev/rbd/bdev_rbd.h b/lib/bdev/rbd/bdev_rbd.h index 398b11a82..c275ba464 100644 --- a/lib/bdev/rbd/bdev_rbd.h +++ b/lib/bdev/rbd/bdev_rbd.h @@ -38,9 +38,13 @@ #include "spdk/bdev.h" +void spdk_bdev_rbd_free_config(char **config); +char **spdk_bdev_rbd_dup_config(const char *const *config); + typedef void (*spdk_delete_rbd_complete)(void *cb_arg, int bdeverrno); -struct spdk_bdev *spdk_bdev_rbd_create(const char *name, const char *pool_name, +struct spdk_bdev *spdk_bdev_rbd_create(const char *name, const char *user_id, const char *pool_name, + const char *const *config, const char *rbd_name, uint32_t block_size); /** * Delete rbd bdev. diff --git a/lib/bdev/rbd/bdev_rbd_rpc.c b/lib/bdev/rbd/bdev_rbd_rpc.c index 745a90ed4..31ff8d989 100644 --- a/lib/bdev/rbd/bdev_rbd_rpc.c +++ b/lib/bdev/rbd/bdev_rbd_rpc.c @@ -39,24 +39,72 @@ struct rpc_construct_rbd { char *name; + char *user_id; char *pool_name; char *rbd_name; uint32_t block_size; + char **config; }; static void free_rpc_construct_rbd(struct rpc_construct_rbd *req) { free(req->name); + free(req->user_id); free(req->pool_name); free(req->rbd_name); + spdk_bdev_rbd_free_config(req->config); +} + +static int +spdk_bdev_rbd_decode_config(const struct spdk_json_val *values, void *out) +{ + char ***map = out; + char **entry; + uint32_t i; + + if (values->type == SPDK_JSON_VAL_NULL) { + /* treated like empty object: empty config */ + *map = calloc(1, sizeof(**map)); + if (!*map) { + return -1; + } + return 0; + } + + if (values->type != SPDK_JSON_VAL_OBJECT_BEGIN) { + return -1; + } + + *map = calloc(values->len + 1, sizeof(**map)); + if (!*map) { + return -1; + } + + for (i = 0, entry = *map; i < values->len;) { + const struct spdk_json_val *name = &values[i + 1]; + const struct spdk_json_val *v = &values[i + 2]; + /* Here we catch errors like invalid types. */ + if (!(entry[0] = spdk_json_strdup(name)) || + !(entry[1] = spdk_json_strdup(v))) { + spdk_bdev_rbd_free_config(*map); + *map = NULL; + return -1; + } + i += 1 + spdk_json_val_len(v); + entry += 2; + } + + return 0; } static const struct spdk_json_object_decoder rpc_construct_rbd_decoders[] = { {"name", offsetof(struct rpc_construct_rbd, name), spdk_json_decode_string, true}, + {"user_id", offsetof(struct rpc_construct_rbd, user_id), spdk_json_decode_string, true}, {"pool_name", offsetof(struct rpc_construct_rbd, pool_name), spdk_json_decode_string}, {"rbd_name", offsetof(struct rpc_construct_rbd, rbd_name), spdk_json_decode_string}, {"block_size", offsetof(struct rpc_construct_rbd, block_size), spdk_json_decode_uint32}, + {"config", offsetof(struct rpc_construct_rbd, config), spdk_bdev_rbd_decode_config, true} }; static void @@ -74,7 +122,10 @@ spdk_rpc_construct_rbd_bdev(struct spdk_jsonrpc_request *request, goto invalid; } - bdev = spdk_bdev_rbd_create(req.name, req.pool_name, req.rbd_name, req.block_size); + bdev = spdk_bdev_rbd_create(req.name, req.user_id, req.pool_name, + (const char *const *)req.config, + req.rbd_name, + req.block_size); if (bdev == NULL) { goto invalid; } diff --git a/scripts/rpc.py b/scripts/rpc.py index 28e0d0a2d..8db2b3dee 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -286,8 +286,18 @@ if __name__ == "__main__": p.set_defaults(func=delete_nvme_controller) def construct_rbd_bdev(args): + config = None + if args.config: + config = {} + for entry in args.config: + parts = entry.split('=', 1) + if len(parts) != 2: + raise Exception('--config %s not in key=value form' % entry) + config[parts[0]] = parts[1] print(rpc.bdev.construct_rbd_bdev(args.client, name=args.name, + user=args.user, + config=config, pool_name=args.pool_name, rbd_name=args.rbd_name, block_size=args.block_size)) @@ -295,6 +305,9 @@ if __name__ == "__main__": p = subparsers.add_parser('construct_rbd_bdev', help='Add a bdev with ceph rbd backend') p.add_argument('-b', '--name', help="Name of the bdev", required=False) + p.add_argument('--user', help="Ceph user name (i.e. admin, not client.admin)", required=False) + p.add_argument('--config', action='append', metavar='key=value', + help="adds a key=value configuration option for rados_conf_set (default: rely on config file)") p.add_argument('pool_name', help='rbd pool name') p.add_argument('rbd_name', help='rbd image name') p.add_argument('block_size', help='rbd block size', type=int) diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 86d1e333c..57a3174b3 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -256,7 +256,7 @@ def delete_nvme_controller(client, name): return client.call('delete_nvme_controller', params) -def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None): +def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None, user=None, config=None): """Construct a Ceph RBD block device. Args: @@ -264,6 +264,8 @@ def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None): rbd_name: Ceph RBD image name block_size: block size of RBD volume name: name of block device (optional) + user: Ceph user name (optional) + config: map of config keys to values (optional) Returns: Name of created block device. @@ -276,6 +278,10 @@ def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None): if name: params['name'] = name + if user is not None: + params['user_id'] = user + if config is not None: + params['config'] = config return client.call('construct_rbd_bdev', params)