From 20c8a3b8db28b35adba9219519b54a384f587319 Mon Sep 17 00:00:00 2001 From: Tan Long Date: Wed, 1 Dec 2021 04:13:32 -0500 Subject: [PATCH] bdev/rbd: Add key_file to the rbd_register_cluster RPC In project practice, config_file and key_file are often used to connect to a rados cluster, config_file includes "mon_host" and other rados configurations like "rbd_cache", and key_file includes the secret key and the access authority to each pool for current user. This patch adds key_file option, user can specify config_file and key_file or only config_param to connect rados cluster. This will make it much more flexible for users with his/her convenience. Signed-off-by: Tan Long Change-Id: I6b49aad70b578bdeb3ac8ea9ca0fcbd931582025 Signed-off-by: Tan Long Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10485 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Reviewed-by: GangCao Reviewed-by: Changpeng Liu Reviewed-by: Xiaodong Liu Reviewed-by: Aleksey Marchuk Tested-by: SPDK CI Jenkins --- CHANGELOG.md | 3 +++ doc/jsonrpc.md | 33 +++++++++++++++++++-------------- module/bdev/rbd/bdev_rbd.c | 30 ++++++++++++++++++++++++++++-- module/bdev/rbd/bdev_rbd.h | 1 + module/bdev/rbd/bdev_rbd_rpc.c | 4 +++- scripts/rpc.py | 4 +++- scripts/rpc/bdev.py | 5 ++++- 7 files changed, 61 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7721c7f81..83fc94da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ removed in SPDK 22.04, and the parameter `transport_retry_count` is added and us An new parameter `bdev_retry_count` is added to the RPC `bdev_nvme_set_options`. +Added 'key_file' parameter to the 'rbd_register_cluster' RPC. It is an optional parameter to +specify a keyring file to connect to a RADOS cluster. + ### nvme New APIs, `spdk_nvme_ctrlr_disconnect`, `spdk_nvme_ctrlr_reconnect_async`, and diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index b91eb33f4..3277b68af 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -3606,37 +3606,42 @@ name | Required | string | Registerd Rados cluster objec user_id | Optional | string | Ceph ID (i.e. admin, not client.admin) config_param | Optional | string map | Explicit librados configuration config_file | Optional | string | File path of libraodos configuration file +key_file | Optional | string | File path of libraodos key file This RPC registers a Rados Cluster object handle which is only known -to rbd module, it uses user_id + config_param or user_id + config_file to -identify a Rados cluster object. - -If no config_param is specified, Ceph configuration files must exist with -all relevant settings for accessing the Ceph cluster. If a config map is -passed, the configuration files are ignored and instead all key/value -pairs are passed to rados_conf_set to configure cluster access. In -practice, "mon_host" (= list of monitor address+port) and "key" (= the -secret key stored in Ceph keyrings) are enough. +to rbd module, it uses user_id + config_param or user_id + config_file + +key_file or user_id + config_param + config_file + key_file to identify +a Rados cluster object. When accessing the Ceph cluster as some user other than "admin" (the default), the "user_id" has to be set. +The configuration items and secret key can be specified by setting config_param, +config_file and key_file, all of them, or none of them. If only config_param is +passed, all key/value pairs are passed to rados_conf_set to configure cluster access. +In practice, "mon_host" (= list of monitor address+port) and "key" (= the secret key +stored in Ceph keyrings) are enough. If config_file and key_file are specified, they must +exist with all relevant settings for accessing the Ceph cluster. If config_param, config_file +and key_file are specified, get the key/value pairs from config_file first and set to +rados_conf_set function, then set pairs in config_param and keyring in key_file. If nothing +is specified, it will get configuration file and key file from the default location +/etc/ceph/ceph.conf and /etc/ceph/ceph.client.user_id.keyring. + #### Result Name of newly created Rados cluster object. #### Example -Example request with `key` from `/etc/ceph/ceph.client.admin.keyring`: +Example request: ~~ { "params": { "name": "rbd_cluster", - "config_param": { - "mon_host": "192.168.7.1:6789,192.168.7.2:6789", - "key": "AQDwf8db7zR1GRAA5k7NKXjS5S5V4mntwUDnGQ==", - } + "user_id": cinder, + "config_file": "/root/ceph_conf/ceph.conf", + "key_file": "/root/ceph_conf/ceph.client.cinder.keyring" }, "jsonrpc": "2.0", "method": "bdev_rbd_register_cluster", diff --git a/module/bdev/rbd/bdev_rbd.c b/module/bdev/rbd/bdev_rbd.c index 791966b11..ac22462f6 100644 --- a/module/bdev/rbd/bdev_rbd.c +++ b/module/bdev/rbd/bdev_rbd.c @@ -93,6 +93,7 @@ struct bdev_rbd_cluster { char *user_id; char **config_param; char *config_file; + char *key_file; rados_t cluster; uint32_t ref; STAILQ_ENTRY(bdev_rbd_cluster) link; @@ -109,6 +110,7 @@ bdev_rbd_cluster_free(struct bdev_rbd_cluster *entry) bdev_rbd_free_config(entry->config_param); free(entry->config_file); + free(entry->key_file); free(entry->user_id); free(entry->name); free(entry); @@ -803,6 +805,9 @@ bdev_rbd_cluster_dump_entry(const char *cluster_name, struct spdk_json_write_ctx } else if (entry->config_file) { spdk_json_write_named_string(w, "config_file", entry->config_file); } + if (entry->key_file) { + spdk_json_write_named_string(w, "key_file", entry->key_file); + } pthread_mutex_unlock(&g_map_bdev_rbd_cluster_mutex); return; @@ -910,6 +915,9 @@ dump_single_cluster_entry(struct bdev_rbd_cluster *entry, struct spdk_json_write } else if (entry->config_file) { spdk_json_write_named_string(w, "config_file", entry->config_file); } + if (entry->key_file) { + spdk_json_write_named_string(w, "key_file", entry->key_file); + } spdk_json_write_object_end(w); } @@ -967,7 +975,7 @@ static const struct spdk_bdev_fn_table rbd_fn_table = { static int rbd_register_cluster(const char *name, const char *user_id, const char *const *config_param, - const char *config_file) + const char *config_file, const char *key_file) { struct bdev_rbd_cluster *entry; int rc; @@ -1017,6 +1025,14 @@ rbd_register_cluster(const char *name, const char *user_id, const char *const *c } } + if (key_file) { + entry->key_file = strdup(key_file); + if (entry->key_file == NULL) { + SPDK_ERRLOG("Failed to save the key_file=%s on entry = %p\n", key_file, entry); + goto err_handle; + } + } + rc = rados_create(&entry->cluster, user_id); if (rc < 0) { SPDK_ERRLOG("Failed to create rados_t struct\n"); @@ -1043,6 +1059,15 @@ rbd_register_cluster(const char *name, const char *user_id, const char *const *c } } + if (key_file) { + rc = rados_conf_set(entry->cluster, "keyring", key_file); + if (rc < 0) { + SPDK_ERRLOG("Failed to set keyring = %s\n", key_file); + rados_shutdown(entry->cluster); + goto err_handle; + } + } + rc = rados_connect(entry->cluster); if (rc < 0) { SPDK_ERRLOG("Failed to connect to rbd_pool on cluster=%p\n", entry->cluster); @@ -1104,7 +1129,8 @@ _bdev_rbd_register_cluster(void *arg) int rc; rc = rbd_register_cluster((const char *)info->name, (const char *)info->user_id, - (const char *const *)info->config_param, (const char *)info->config_file); + (const char *const *)info->config_param, (const char *)info->config_file, + (const char *)info->key_file); if (rc) { ret = NULL; } diff --git a/module/bdev/rbd/bdev_rbd.h b/module/bdev/rbd/bdev_rbd.h index 1deef34cc..8f342ec5b 100644 --- a/module/bdev/rbd/bdev_rbd.h +++ b/module/bdev/rbd/bdev_rbd.h @@ -44,6 +44,7 @@ struct cluster_register_info { char *user_id; char **config_param; char *config_file; + char *key_file; }; void bdev_rbd_free_config(char **config); diff --git a/module/bdev/rbd/bdev_rbd_rpc.c b/module/bdev/rbd/bdev_rbd_rpc.c index cc07fffcb..912d5cc82 100644 --- a/module/bdev/rbd/bdev_rbd_rpc.c +++ b/module/bdev/rbd/bdev_rbd_rpc.c @@ -269,13 +269,15 @@ free_rpc_register_cluster(struct cluster_register_info *req) free(req->user_id); bdev_rbd_free_config(req->config_param); free(req->config_file); + free(req->key_file); } static const struct spdk_json_object_decoder rpc_register_cluster_decoders[] = { {"name", offsetof(struct cluster_register_info, name), spdk_json_decode_string, true}, {"user_id", offsetof(struct cluster_register_info, user_id), spdk_json_decode_string, true}, {"config_param", offsetof(struct cluster_register_info, config_param), bdev_rbd_decode_config, true}, - {"config_file", offsetof(struct cluster_register_info, config_file), spdk_json_decode_string, true} + {"config_file", offsetof(struct cluster_register_info, config_file), spdk_json_decode_string, true}, + {"key_file", offsetof(struct cluster_register_info, key_file), spdk_json_decode_string, true} }; static void diff --git a/scripts/rpc.py b/scripts/rpc.py index 7ca2190c1..014dc4124 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -704,7 +704,8 @@ if __name__ == "__main__": name=args.name, user=args.user, config_param=config_param, - config_file=args.config_file)) + config_file=args.config_file, + key_file=args.key_file)) p = subparsers.add_parser('bdev_rbd_register_cluster', help='Add a Rados cluster with ceph rbd backend') @@ -713,6 +714,7 @@ if __name__ == "__main__": p.add_argument('--config-param', action='append', metavar='key=value', help="adds a key=value configuration option for rados_conf_set (default: rely on config file)") p.add_argument('--config-file', help="The file path of the Rados configuration file", required=False) + p.add_argument('--key-file', help="The file path of the Rados keyring file", required=False) p.set_defaults(func=bdev_rbd_register_cluster) def bdev_rbd_unregister_cluster(args): diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index f470784ce..6bed18bc8 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -747,7 +747,7 @@ def bdev_zone_block_delete(client, name): return client.call('bdev_zone_block_delete', params) -def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config_file=None): +def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config_file=None, key_file=None): """Create a Rados Cluster object of the Ceph RBD backend. Args: @@ -755,6 +755,7 @@ def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config user: Ceph user name (optional) config_param: map of config keys to values (optional) config_file: file path of Ceph configuration file (optional) + key_file: file path of Ceph key file (optional) Returns: Name of registered Rados Cluster object. @@ -767,6 +768,8 @@ def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config params['config_param'] = config_param if config_file is not None: params['config_file'] = config_file + if key_file is not None: + params['key_file'] = key_file return client.call('bdev_rbd_register_cluster', params)