bdev/rbd: full control over client configuration
When the caller of the RPC API has all the necessary information about how to access a Ceph cluster, then having to create configuration files before calling the RPC API is problematic (has to touch files owned by a local admin, changes must be removed again). But having to encode support for certain configuration options in SPDK is also problematic, because that might change depending on the librados version. The approach taken here is to merely pass through arbitrary key/value config options. Existing config files are ignored when that happens. The caller of the RPC then has full control over the connection setup and can be sure that he does not inherit settings from a local file accidentally. In addition, user management is supported now, with or without a config. This is useful for accessing a volume with a less privileged user. Previously, passing NULL to rados_create implicitly chose the "admin" user. Change-Id: I5e7f36092df663a3d7ac503c04fc624a8fe1208e Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> Reviewed-on: https://review.gerrithub.io/430460 Reviewed-by: Pawel Wodkowski <pawelx.wodkowski@intel.com> Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Patrick Ohly
Jim Harris
parent
38dfce0428
commit
4cfae03606
@ -1160,9 +1160,21 @@ This method is available only if SPDK was build with Ceph RBD support.
|
||||
Name | Optional | Type | Description
|
||||
----------------------- | -------- | ----------- | -----------
|
||||
name | Optional | string | Bdev name
|
||||
user_id | Optional | string | Ceph ID (i.e. admin, not client.admin)
|
||||
pool_name | Required | string | Pool name
|
||||
rbd_name | Required | string | Image name
|
||||
block_size | Required | number | Block size
|
||||
config | Optional | string map | Explicit librados configuration
|
||||
|
||||
If no config is specified, Ceph configuration files must exist with
|
||||
all relevant settings for accessing the pool. If a config map is
|
||||
passed, the configuration files are ignored and instead all key/value
|
||||
pairs are passed to rados_conf_set to configure cluster access. In
|
||||
practice, "mon_host" (= list of monitor address+port) and "key" (= the
|
||||
secret key stored in Ceph keyrings) are enough.
|
||||
|
||||
When accessing the image as some user other than "admin" (the
|
||||
default), the "user_id" has to be set.
|
||||
|
||||
### Result
|
||||
|
||||
@ -1170,13 +1182,17 @@ Name of newly created bdev.
|
||||
|
||||
### Example
|
||||
|
||||
Example request:
|
||||
Example request with `key` from `/etc/ceph/ceph.client.admin.keyring`:
|
||||
|
||||
~~~
|
||||
{
|
||||
"params": {
|
||||
"pool_name": "rbd",
|
||||
"rbd_name": "foo",
|
||||
"config": {
|
||||
"mon_host": "192.168.7.1:6789,192.168.7.2:6789",
|
||||
"key": "AQDwf8db7zR1GRAA5k7NKXjS5S5V4mntwUDnGQ==",
|
||||
}
|
||||
"block_size": 4096
|
||||
},
|
||||
"jsonrpc": "2.0",
|
||||
|
||||
@ -59,7 +59,9 @@ static int bdev_rbd_count = 0;
|
||||
struct bdev_rbd {
|
||||
struct spdk_bdev disk;
|
||||
char *rbd_name;
|
||||
char *user_id;
|
||||
char *pool_name;
|
||||
char **config;
|
||||
rbd_image_info_t info;
|
||||
TAILQ_ENTRY(bdev_rbd) tailq;
|
||||
struct spdk_poller *reset_timer;
|
||||
@ -90,27 +92,78 @@ bdev_rbd_free(struct bdev_rbd *rbd)
|
||||
|
||||
free(rbd->disk.name);
|
||||
free(rbd->rbd_name);
|
||||
free(rbd->user_id);
|
||||
free(rbd->pool_name);
|
||||
spdk_bdev_rbd_free_config(rbd->config);
|
||||
free(rbd);
|
||||
}
|
||||
|
||||
void
|
||||
spdk_bdev_rbd_free_config(char **config)
|
||||
{
|
||||
char **entry;
|
||||
|
||||
if (config) {
|
||||
for (entry = config; *entry; entry++) {
|
||||
free(*entry);
|
||||
}
|
||||
free(config);
|
||||
}
|
||||
}
|
||||
|
||||
char **
|
||||
spdk_bdev_rbd_dup_config(const char *const *config)
|
||||
{
|
||||
size_t count;
|
||||
char **copy;
|
||||
|
||||
if (!config) {
|
||||
return NULL;
|
||||
}
|
||||
for (count = 0; config[count]; count++) {}
|
||||
copy = calloc(count + 1, sizeof(*copy));
|
||||
if (!copy) {
|
||||
return NULL;
|
||||
}
|
||||
for (count = 0; config[count]; count++) {
|
||||
if (!(copy[count] = strdup(config[count]))) {
|
||||
spdk_bdev_rbd_free_config(copy);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
return copy;
|
||||
}
|
||||
|
||||
static int
|
||||
bdev_rados_context_init(const char *rbd_pool_name, rados_t *cluster,
|
||||
rados_ioctx_t *io_ctx)
|
||||
bdev_rados_context_init(const char *user_id, const char *rbd_pool_name, const char *const *config,
|
||||
rados_t *cluster, rados_ioctx_t *io_ctx)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = rados_create(cluster, NULL);
|
||||
ret = rados_create(cluster, user_id);
|
||||
if (ret < 0) {
|
||||
SPDK_ERRLOG("Failed to create rados_t struct\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
ret = rados_conf_read_file(*cluster, NULL);
|
||||
if (ret < 0) {
|
||||
SPDK_ERRLOG("Failed to read conf file\n");
|
||||
rados_shutdown(*cluster);
|
||||
return -1;
|
||||
if (config) {
|
||||
const char *const *entry = config;
|
||||
while (*entry) {
|
||||
ret = rados_conf_set(*cluster, entry[0], entry[1]);
|
||||
if (ret < 0) {
|
||||
SPDK_ERRLOG("Failed to set %s = %s\n", entry[0], entry[1]);
|
||||
rados_shutdown(*cluster);
|
||||
return -1;
|
||||
}
|
||||
entry += 2;
|
||||
}
|
||||
} else {
|
||||
ret = rados_conf_read_file(*cluster, NULL);
|
||||
if (ret < 0) {
|
||||
SPDK_ERRLOG("Failed to read conf file\n");
|
||||
rados_shutdown(*cluster);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
ret = rados_connect(*cluster);
|
||||
@ -132,17 +185,18 @@ bdev_rados_context_init(const char *rbd_pool_name, rados_t *cluster,
|
||||
}
|
||||
|
||||
static int
|
||||
bdev_rbd_init(const char *rbd_pool_name, const char *rbd_name, rbd_image_info_t *info)
|
||||
bdev_rbd_init(const char *user_id, const char *rbd_pool_name, const char *const *config,
|
||||
const char *rbd_name, rbd_image_info_t *info)
|
||||
{
|
||||
int ret;
|
||||
rados_t cluster = NULL;
|
||||
rados_ioctx_t io_ctx = NULL;
|
||||
rbd_image_t image = NULL;
|
||||
|
||||
ret = bdev_rados_context_init(rbd_pool_name, &cluster, &io_ctx);
|
||||
ret = bdev_rados_context_init(user_id, rbd_pool_name, config, &cluster, &io_ctx);
|
||||
if (ret < 0) {
|
||||
SPDK_ERRLOG("Failed to create rados context for rbd_pool=%s\n",
|
||||
rbd_pool_name);
|
||||
SPDK_ERRLOG("Failed to create rados context for user_id=%s and rbd_pool=%s\n",
|
||||
user_id ? user_id : "admin (the default)", rbd_pool_name);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@ -493,10 +547,12 @@ bdev_rbd_create_cb(void *io_device, void *ctx_buf)
|
||||
ch->io_ctx = NULL;
|
||||
ch->pfd.fd = -1;
|
||||
|
||||
ret = bdev_rados_context_init(ch->disk->pool_name, &ch->cluster, &ch->io_ctx);
|
||||
ret = bdev_rados_context_init(ch->disk->user_id, ch->disk->pool_name,
|
||||
(const char *const *)ch->disk->config,
|
||||
&ch->cluster, &ch->io_ctx);
|
||||
if (ret < 0) {
|
||||
SPDK_ERRLOG("Failed to create rados context for rbd_pool=%s\n",
|
||||
ch->disk->pool_name);
|
||||
SPDK_ERRLOG("Failed to create rados context for user_id %s and rbd_pool=%s\n",
|
||||
ch->disk->user_id ? ch->disk->user_id : "admin (the default)", ch->disk->pool_name);
|
||||
goto err;
|
||||
}
|
||||
|
||||
@ -558,6 +614,22 @@ bdev_rbd_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
|
||||
spdk_json_write_name(w, "rbd_name");
|
||||
spdk_json_write_string(w, rbd_bdev->rbd_name);
|
||||
|
||||
if (rbd_bdev->user_id) {
|
||||
spdk_json_write_named_string(w, "user_id", rbd_bdev->user_id);
|
||||
}
|
||||
|
||||
if (rbd_bdev->config) {
|
||||
char **entry = rbd_bdev->config;
|
||||
|
||||
spdk_json_write_name(w, "config");
|
||||
spdk_json_write_object_begin(w);
|
||||
while (*entry) {
|
||||
spdk_json_write_named_string(w, entry[0], entry[1]);
|
||||
entry += 2;
|
||||
}
|
||||
spdk_json_write_object_end(w);
|
||||
}
|
||||
|
||||
spdk_json_write_object_end(w);
|
||||
|
||||
return 0;
|
||||
@ -577,6 +649,22 @@ bdev_rbd_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w
|
||||
spdk_json_write_named_string(w, "pool_name", rbd->pool_name);
|
||||
spdk_json_write_named_string(w, "rbd_name", rbd->rbd_name);
|
||||
spdk_json_write_named_uint32(w, "block_size", bdev->blocklen);
|
||||
if (rbd->user_id) {
|
||||
spdk_json_write_named_string(w, "user_id", rbd->user_id);
|
||||
}
|
||||
|
||||
if (rbd->config) {
|
||||
char **entry = rbd->config;
|
||||
|
||||
spdk_json_write_name(w, "config");
|
||||
spdk_json_write_object_begin(w);
|
||||
while (*entry) {
|
||||
spdk_json_write_named_string(w, entry[0], entry[1]);
|
||||
entry += 2;
|
||||
}
|
||||
spdk_json_write_object_end(w);
|
||||
}
|
||||
|
||||
spdk_json_write_object_end(w);
|
||||
|
||||
spdk_json_write_object_end(w);
|
||||
@ -592,7 +680,9 @@ static const struct spdk_bdev_fn_table rbd_fn_table = {
|
||||
};
|
||||
|
||||
struct spdk_bdev *
|
||||
spdk_bdev_rbd_create(const char *name, const char *pool_name, const char *rbd_name,
|
||||
spdk_bdev_rbd_create(const char *name, const char *user_id, const char *pool_name,
|
||||
const char *const *config,
|
||||
const char *rbd_name,
|
||||
uint32_t block_size)
|
||||
{
|
||||
struct bdev_rbd *rbd;
|
||||
@ -614,13 +704,28 @@ spdk_bdev_rbd_create(const char *name, const char *pool_name, const char *rbd_na
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
rbd->user_id = strdup(user_id);
|
||||
if (!rbd->user_id) {
|
||||
bdev_rbd_free(rbd);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
rbd->pool_name = strdup(pool_name);
|
||||
if (!rbd->pool_name) {
|
||||
bdev_rbd_free(rbd);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = bdev_rbd_init(rbd->pool_name, rbd_name, &rbd->info);
|
||||
if (config && !(rbd->config = spdk_bdev_rbd_dup_config(config))) {
|
||||
bdev_rbd_free(rbd);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = bdev_rbd_init(rbd->user_id, rbd->pool_name,
|
||||
(const char *const *)rbd->config,
|
||||
rbd_name, &rbd->info);
|
||||
if (ret < 0) {
|
||||
bdev_rbd_free(rbd);
|
||||
SPDK_ERRLOG("Failed to init rbd device\n");
|
||||
@ -727,7 +832,8 @@ bdev_rbd_library_init(void)
|
||||
}
|
||||
}
|
||||
|
||||
if (spdk_bdev_rbd_create(NULL, pool_name, rbd_name, block_size) == NULL) {
|
||||
/* TODO(?): user_id and rbd config values */
|
||||
if (spdk_bdev_rbd_create(NULL, NULL, pool_name, NULL, rbd_name, block_size) == NULL) {
|
||||
rc = -1;
|
||||
goto end;
|
||||
}
|
||||
|
||||
@ -38,9 +38,13 @@
|
||||
|
||||
#include "spdk/bdev.h"
|
||||
|
||||
void spdk_bdev_rbd_free_config(char **config);
|
||||
char **spdk_bdev_rbd_dup_config(const char *const *config);
|
||||
|
||||
typedef void (*spdk_delete_rbd_complete)(void *cb_arg, int bdeverrno);
|
||||
|
||||
struct spdk_bdev *spdk_bdev_rbd_create(const char *name, const char *pool_name,
|
||||
struct spdk_bdev *spdk_bdev_rbd_create(const char *name, const char *user_id, const char *pool_name,
|
||||
const char *const *config,
|
||||
const char *rbd_name, uint32_t block_size);
|
||||
/**
|
||||
* Delete rbd bdev.
|
||||
|
||||
@ -39,24 +39,72 @@
|
||||
|
||||
struct rpc_construct_rbd {
|
||||
char *name;
|
||||
char *user_id;
|
||||
char *pool_name;
|
||||
char *rbd_name;
|
||||
uint32_t block_size;
|
||||
char **config;
|
||||
};
|
||||
|
||||
static void
|
||||
free_rpc_construct_rbd(struct rpc_construct_rbd *req)
|
||||
{
|
||||
free(req->name);
|
||||
free(req->user_id);
|
||||
free(req->pool_name);
|
||||
free(req->rbd_name);
|
||||
spdk_bdev_rbd_free_config(req->config);
|
||||
}
|
||||
|
||||
static int
|
||||
spdk_bdev_rbd_decode_config(const struct spdk_json_val *values, void *out)
|
||||
{
|
||||
char ***map = out;
|
||||
char **entry;
|
||||
uint32_t i;
|
||||
|
||||
if (values->type == SPDK_JSON_VAL_NULL) {
|
||||
/* treated like empty object: empty config */
|
||||
*map = calloc(1, sizeof(**map));
|
||||
if (!*map) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (values->type != SPDK_JSON_VAL_OBJECT_BEGIN) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
*map = calloc(values->len + 1, sizeof(**map));
|
||||
if (!*map) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (i = 0, entry = *map; i < values->len;) {
|
||||
const struct spdk_json_val *name = &values[i + 1];
|
||||
const struct spdk_json_val *v = &values[i + 2];
|
||||
/* Here we catch errors like invalid types. */
|
||||
if (!(entry[0] = spdk_json_strdup(name)) ||
|
||||
!(entry[1] = spdk_json_strdup(v))) {
|
||||
spdk_bdev_rbd_free_config(*map);
|
||||
*map = NULL;
|
||||
return -1;
|
||||
}
|
||||
i += 1 + spdk_json_val_len(v);
|
||||
entry += 2;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct spdk_json_object_decoder rpc_construct_rbd_decoders[] = {
|
||||
{"name", offsetof(struct rpc_construct_rbd, name), spdk_json_decode_string, true},
|
||||
{"user_id", offsetof(struct rpc_construct_rbd, user_id), spdk_json_decode_string, true},
|
||||
{"pool_name", offsetof(struct rpc_construct_rbd, pool_name), spdk_json_decode_string},
|
||||
{"rbd_name", offsetof(struct rpc_construct_rbd, rbd_name), spdk_json_decode_string},
|
||||
{"block_size", offsetof(struct rpc_construct_rbd, block_size), spdk_json_decode_uint32},
|
||||
{"config", offsetof(struct rpc_construct_rbd, config), spdk_bdev_rbd_decode_config, true}
|
||||
};
|
||||
|
||||
static void
|
||||
@ -74,7 +122,10 @@ spdk_rpc_construct_rbd_bdev(struct spdk_jsonrpc_request *request,
|
||||
goto invalid;
|
||||
}
|
||||
|
||||
bdev = spdk_bdev_rbd_create(req.name, req.pool_name, req.rbd_name, req.block_size);
|
||||
bdev = spdk_bdev_rbd_create(req.name, req.user_id, req.pool_name,
|
||||
(const char *const *)req.config,
|
||||
req.rbd_name,
|
||||
req.block_size);
|
||||
if (bdev == NULL) {
|
||||
goto invalid;
|
||||
}
|
||||
|
||||
@ -286,8 +286,18 @@ if __name__ == "__main__":
|
||||
p.set_defaults(func=delete_nvme_controller)
|
||||
|
||||
def construct_rbd_bdev(args):
|
||||
config = None
|
||||
if args.config:
|
||||
config = {}
|
||||
for entry in args.config:
|
||||
parts = entry.split('=', 1)
|
||||
if len(parts) != 2:
|
||||
raise Exception('--config %s not in key=value form' % entry)
|
||||
config[parts[0]] = parts[1]
|
||||
print(rpc.bdev.construct_rbd_bdev(args.client,
|
||||
name=args.name,
|
||||
user=args.user,
|
||||
config=config,
|
||||
pool_name=args.pool_name,
|
||||
rbd_name=args.rbd_name,
|
||||
block_size=args.block_size))
|
||||
@ -295,6 +305,9 @@ if __name__ == "__main__":
|
||||
p = subparsers.add_parser('construct_rbd_bdev',
|
||||
help='Add a bdev with ceph rbd backend')
|
||||
p.add_argument('-b', '--name', help="Name of the bdev", required=False)
|
||||
p.add_argument('--user', help="Ceph user name (i.e. admin, not client.admin)", required=False)
|
||||
p.add_argument('--config', action='append', metavar='key=value',
|
||||
help="adds a key=value configuration option for rados_conf_set (default: rely on config file)")
|
||||
p.add_argument('pool_name', help='rbd pool name')
|
||||
p.add_argument('rbd_name', help='rbd image name')
|
||||
p.add_argument('block_size', help='rbd block size', type=int)
|
||||
|
||||
@ -256,7 +256,7 @@ def delete_nvme_controller(client, name):
|
||||
return client.call('delete_nvme_controller', params)
|
||||
|
||||
|
||||
def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None):
|
||||
def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None, user=None, config=None):
|
||||
"""Construct a Ceph RBD block device.
|
||||
|
||||
Args:
|
||||
@ -264,6 +264,8 @@ def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None):
|
||||
rbd_name: Ceph RBD image name
|
||||
block_size: block size of RBD volume
|
||||
name: name of block device (optional)
|
||||
user: Ceph user name (optional)
|
||||
config: map of config keys to values (optional)
|
||||
|
||||
Returns:
|
||||
Name of created block device.
|
||||
@ -276,6 +278,10 @@ def construct_rbd_bdev(client, pool_name, rbd_name, block_size, name=None):
|
||||
|
||||
if name:
|
||||
params['name'] = name
|
||||
if user is not None:
|
||||
params['user_id'] = user
|
||||
if config is not None:
|
||||
params['config'] = config
|
||||
|
||||
return client.call('construct_rbd_bdev', params)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user