From 1a6dac405b14b167ca94466e623db9110116e19c Mon Sep 17 00:00:00 2001 From: Pawel Wodkowski Date: Tue, 27 Mar 2018 15:45:10 +0200 Subject: [PATCH] bdev/virito: add JSON config dump and generic create RPC The new construct_virtio_dev allow creating virtio SCSI and blk for both PCI and user transports. Change-Id: Ibd79c4fb75e3cbd993b46227d86e915c1b740a18 Signed-off-by: Pawel Wodkowski Reviewed-on: https://review.gerrithub.io/405419 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker --- include/spdk_internal/virtio.h | 5 +- lib/bdev/virtio/bdev_virtio_blk.c | 24 ++++- lib/bdev/virtio/bdev_virtio_rpc.c | 156 ++++++++++++++++++++++++++++- lib/bdev/virtio/bdev_virtio_scsi.c | 47 ++++++++- lib/virtio/virtio.c | 4 +- lib/virtio/virtio_pci.c | 18 +++- lib/virtio/virtio_user.c | 16 ++- scripts/rpc.py | 18 ++++ scripts/rpc/vhost.py | 14 +++ 9 files changed, 285 insertions(+), 17 deletions(-) diff --git a/include/spdk_internal/virtio.h b/include/spdk_internal/virtio.h index 076b91b00..66942b863 100644 --- a/include/spdk_internal/virtio.h +++ b/include/spdk_internal/virtio.h @@ -122,7 +122,8 @@ struct virtio_dev_ops { void (*del_queue)(struct virtio_dev *hw, struct virtqueue *vq); void (*notify_queue)(struct virtio_dev *hw, struct virtqueue *vq); - void (*dump_json_config)(struct virtio_dev *hw, struct spdk_json_write_ctx *w); + void (*dump_json_info)(struct virtio_dev *hw, struct spdk_json_write_ctx *w); + void (*write_json_config)(struct virtio_dev *hw, struct spdk_json_write_ctx *w); }; struct vq_desc_extra { @@ -424,7 +425,7 @@ virtio_dev_has_feature(struct virtio_dev *vdev, uint64_t bit) * \param vdev virtio device * \param w json stream */ -void virtio_dev_dump_json_config(struct virtio_dev *vdev, struct spdk_json_write_ctx *w); +void virtio_dev_dump_json_info(struct virtio_dev *vdev, struct spdk_json_write_ctx *w); /** * Enumerate all PCI Virtio devices of given type on the system. diff --git a/lib/bdev/virtio/bdev_virtio_blk.c b/lib/bdev/virtio/bdev_virtio_blk.c index 30b70d8ba..fdfd2a91c 100644 --- a/lib/bdev/virtio/bdev_virtio_blk.c +++ b/lib/bdev/virtio/bdev_virtio_blk.c @@ -244,16 +244,38 @@ bdev_virtio_dump_json_config(void *ctx, struct spdk_json_write_ctx *w) { struct virtio_blk_dev *bvdev = ctx; - virtio_dev_dump_json_config(&bvdev->vdev, w); + virtio_dev_dump_json_info(&bvdev->vdev, w); return 0; } +static void +bdev_virtio_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +{ + struct virtio_blk_dev *bvdev = bdev->ctxt; + + spdk_json_write_object_begin(w); + + spdk_json_write_named_string(w, "method", "construct_virtio_dev"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "name", bvdev->vdev.name); + spdk_json_write_named_string(w, "dev_type", "blk"); + + /* Write transport specific parameters. */ + bvdev->vdev.backend_ops->write_json_config(&bvdev->vdev, w); + + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); +} + static const struct spdk_bdev_fn_table virtio_fn_table = { .destruct = bdev_virtio_disk_destruct, .submit_request = bdev_virtio_submit_request, .io_type_supported = bdev_virtio_io_type_supported, .get_io_channel = bdev_virtio_get_io_channel, .dump_info_json = bdev_virtio_dump_json_config, + .write_config_json = bdev_virtio_write_config_json, }; static void diff --git a/lib/bdev/virtio/bdev_virtio_rpc.c b/lib/bdev/virtio/bdev_virtio_rpc.c index 589bc87a2..286465552 100644 --- a/lib/bdev/virtio/bdev_virtio_rpc.c +++ b/lib/bdev/virtio/bdev_virtio_rpc.c @@ -40,6 +40,9 @@ #include "bdev_virtio.h" +#define SPDK_VIRTIO_USER_DEFAULT_VQ_COUNT 1 +#define SPDK_VIRTIO_USER_DEFAULT_QUEUE_SIZE 512 + struct rpc_construct_virtio_scsi_dev { char *path; char *pci_address; @@ -113,8 +116,8 @@ spdk_rpc_create_virtio_user_scsi_bdev(struct spdk_jsonrpc_request *request, } req->pci_address = NULL; - req->vq_count = 1; - req->vq_size = 512; + req->vq_count = SPDK_VIRTIO_USER_DEFAULT_VQ_COUNT; + req->vq_size = SPDK_VIRTIO_USER_DEFAULT_QUEUE_SIZE; if (spdk_json_decode_object(params, rpc_construct_virtio_user_scsi_dev, SPDK_COUNTOF(rpc_construct_virtio_user_scsi_dev), @@ -320,8 +323,8 @@ spdk_rpc_create_virtio_user_blk_bdev(struct spdk_jsonrpc_request *request, int rc; req.pci_address = NULL; - req.vq_count = 1; - req.vq_size = 512; + req.vq_count = SPDK_VIRTIO_USER_DEFAULT_VQ_COUNT; + req.vq_size = SPDK_VIRTIO_USER_DEFAULT_QUEUE_SIZE; if (spdk_json_decode_object(params, rpc_construct_virtio_user_blk_dev, SPDK_COUNTOF(rpc_construct_virtio_user_blk_dev), @@ -407,3 +410,148 @@ invalid: free_rpc_construct_virtio_blk_dev(&req); } SPDK_RPC_REGISTER("construct_virtio_pci_blk_bdev", spdk_rpc_create_virtio_pci_blk_bdev); + +struct rpc_construct_virtio_dev { + char *name; + char *trtype; + char *traddr; + char *dev_type; + uint32_t vq_count; + uint32_t vq_size; + struct spdk_jsonrpc_request *request; +}; + +static const struct spdk_json_object_decoder rpc_construct_virtio_dev[] = { + {"name", offsetof(struct rpc_construct_virtio_dev, name), spdk_json_decode_string }, + {"trtype", offsetof(struct rpc_construct_virtio_dev, trtype), spdk_json_decode_string }, + {"traddr", offsetof(struct rpc_construct_virtio_dev, traddr), spdk_json_decode_string }, + {"dev_type", offsetof(struct rpc_construct_virtio_dev, dev_type), spdk_json_decode_string }, + {"vq_count", offsetof(struct rpc_construct_virtio_dev, vq_count), spdk_json_decode_uint32, true }, + {"vq_size", offsetof(struct rpc_construct_virtio_dev, vq_size), spdk_json_decode_uint32, true }, +}; + +static void +free_rpc_construct_virtio_dev(struct rpc_construct_virtio_dev *req) +{ + free(req->name); + free(req->trtype); + free(req->traddr); + free(req->dev_type); + free(req); +} + +static void +spdk_rpc_create_virtio_dev_cb(void *ctx, int result, struct spdk_bdev **bdevs, size_t cnt) +{ + struct rpc_construct_virtio_dev *req = ctx; + struct spdk_json_write_ctx *w; + size_t i; + + if (result) { + spdk_jsonrpc_send_error_response(req->request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(-result)); + free_rpc_construct_virtio_dev(req); + return; + } + + w = spdk_jsonrpc_begin_result(req->request); + if (w) { + spdk_json_write_array_begin(w); + + for (i = 0; i < cnt; i++) { + spdk_json_write_string(w, spdk_bdev_get_name(bdevs[i])); + } + + spdk_json_write_array_end(w); + spdk_jsonrpc_end_result(req->request, w); + } + + free_rpc_construct_virtio_dev(ctx); +} + +static void +spdk_rpc_create_virtio_dev(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_construct_virtio_dev *req; + struct spdk_bdev *bdev; + struct spdk_pci_addr pci_addr; + bool pci; + int rc; + + req = calloc(1, sizeof(*req)); + if (!req) { + SPDK_ERRLOG("calloc() failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, spdk_strerror(ENOMEM)); + return; + } + + if (spdk_json_decode_object(params, rpc_construct_virtio_dev, + SPDK_COUNTOF(rpc_construct_virtio_dev), + req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, spdk_strerror(EINVAL)); + goto invalid; + } + + if (strcmp(req->trtype, "pci") == 0) { + if (req->vq_count != 0 || req->vq_size != 0) { + SPDK_ERRLOG("VQ count or size is not allowed for PCI transport type\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "vq_count or vq_size is not allowed for PCI transport type."); + goto invalid; + } + + if (spdk_pci_addr_parse(&pci_addr, req->traddr) != 0) { + SPDK_ERRLOG("Invalid PCI address '%s'\n", req->traddr); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid PCI address '%s'", req->traddr); + goto invalid; + } + + pci = true; + } else if (strcmp(req->trtype, "user") == 0) { + req->vq_count = req->vq_count == 0 ? SPDK_VIRTIO_USER_DEFAULT_VQ_COUNT : req->vq_count; + req->vq_size = req->vq_size == 0 ? SPDK_VIRTIO_USER_DEFAULT_QUEUE_SIZE : req->vq_size; + pci = false; + } else { + SPDK_ERRLOG("Invalid trtype '%s'\n", req->trtype); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid trtype '%s'", req->trtype); + goto invalid; + } + + req->request = request; + if (strcmp(req->dev_type, "blk") == 0) { + if (pci) { + bdev = bdev_virtio_pci_blk_dev_create(req->name, &pci_addr); + } else { + bdev = bdev_virtio_user_blk_dev_create(req->name, req->traddr, req->vq_count, req->vq_size); + } + + /* Virtio blk doesn't use callback so call it manually to send result. */ + rc = bdev ? 0 : -EINVAL; + spdk_rpc_create_virtio_dev_cb(req, rc, &bdev, bdev ? 1 : 0); + } else if (strcmp(req->dev_type, "scsi") == 0) { + if (pci) { + rc = bdev_virtio_pci_scsi_dev_create(req->name, &pci_addr, spdk_rpc_create_virtio_dev_cb, req); + } else { + rc = bdev_virtio_user_scsi_dev_create(req->name, req->traddr, req->vq_count, req->vq_size, + spdk_rpc_create_virtio_dev_cb, req); + } + + if (rc < 0) { + /* In case of error callback is not called so do it manually to send result. */ + spdk_rpc_create_virtio_dev_cb(req, rc, NULL, 0); + } + } else { + SPDK_ERRLOG("Invalid dev_type '%s'\n", req->dev_type); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid dev_type '%s'", req->dev_type); + goto invalid; + } + + return; +invalid: + free_rpc_construct_virtio_dev(req); +} +SPDK_RPC_REGISTER("construct_virtio_dev", spdk_rpc_create_virtio_dev); diff --git a/lib/bdev/virtio/bdev_virtio_scsi.c b/lib/bdev/virtio/bdev_virtio_scsi.c index 86204b2a0..0b41353d4 100644 --- a/lib/bdev/virtio/bdev_virtio_scsi.c +++ b/lib/bdev/virtio/bdev_virtio_scsi.c @@ -408,11 +408,41 @@ bdev_virtio_get_ctx_size(void) return sizeof(struct virtio_scsi_io_ctx); } +static int +bdev_virtio_scsi_config_json(struct spdk_json_write_ctx *w) +{ + struct virtio_scsi_dev *svdev; + + pthread_mutex_lock(&g_virtio_scsi_mutex); + TAILQ_FOREACH(svdev, &g_virtio_scsi_devs, tailq) { + spdk_json_write_object_begin(w); + + spdk_json_write_named_string(w, "method", "construct_virtio_dev"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "name", svdev->vdev.name); + spdk_json_write_named_string(w, "dev_type", "scsi"); + + /* Write transport specific parameters. */ + svdev->vdev.backend_ops->write_json_config(&svdev->vdev, w); + + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + + } + pthread_mutex_unlock(&g_virtio_scsi_mutex); + + return 0; +} + + static struct spdk_bdev_module virtio_scsi_if = { .name = "virtio_scsi", .module_init = bdev_virtio_initialize, .module_fini = bdev_virtio_finish, .get_ctx_size = bdev_virtio_get_ctx_size, + .config_json = bdev_virtio_scsi_config_json, .async_init = true, .async_fini = true, }; @@ -675,20 +705,29 @@ bdev_virtio_disk_destruct(void *ctx) } static int -bdev_virtio_dump_info_config(void *ctx, struct spdk_json_write_ctx *w) +bdev_virtio_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) { struct virtio_scsi_disk *disk = ctx; - virtio_dev_dump_json_config(&disk->svdev->vdev, w); + virtio_dev_dump_json_info(&disk->svdev->vdev, w); return 0; } +static void +bdev_virtio_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +{ + /* SCSI targets and LUNS are discovered during scan process so nothing + * to save here. + */ +} + static const struct spdk_bdev_fn_table virtio_fn_table = { .destruct = bdev_virtio_disk_destruct, .submit_request = bdev_virtio_submit_request, .io_type_supported = bdev_virtio_io_type_supported, .get_io_channel = bdev_virtio_get_io_channel, - .dump_info_json = bdev_virtio_dump_info_config, + .dump_info_json = bdev_virtio_dump_info_json, + .write_config_json = bdev_virtio_write_config_json, }; static void @@ -1948,7 +1987,7 @@ bdev_virtio_scsi_dev_list(struct spdk_json_write_ctx *w) spdk_json_write_name(w, "name"); spdk_json_write_string(w, svdev->vdev.name); - virtio_dev_dump_json_config(&svdev->vdev, w); + virtio_dev_dump_json_info(&svdev->vdev, w); spdk_json_write_object_end(w); } diff --git a/lib/virtio/virtio.c b/lib/virtio/virtio.c index fa8d8d819..e09f29fb0 100644 --- a/lib/virtio/virtio.c +++ b/lib/virtio/virtio.c @@ -708,7 +708,7 @@ virtio_dev_backend_ops(struct virtio_dev *dev) } void -virtio_dev_dump_json_config(struct virtio_dev *hw, struct spdk_json_write_ctx *w) +virtio_dev_dump_json_info(struct virtio_dev *hw, struct spdk_json_write_ctx *w) { spdk_json_write_name(w, "virtio"); spdk_json_write_object_begin(w); @@ -719,7 +719,7 @@ virtio_dev_dump_json_config(struct virtio_dev *hw, struct spdk_json_write_ctx *w spdk_json_write_name(w, "vq_size"); spdk_json_write_uint32(w, virtio_dev_backend_ops(hw)->get_queue_num(hw, 0)); - virtio_dev_backend_ops(hw)->dump_json_config(hw, w); + virtio_dev_backend_ops(hw)->dump_json_info(hw, w); spdk_json_write_object_end(w); } diff --git a/lib/virtio/virtio_pci.c b/lib/virtio/virtio_pci.c index bf8bcdbcb..73c1bef4d 100644 --- a/lib/virtio/virtio_pci.c +++ b/lib/virtio/virtio_pci.c @@ -108,7 +108,7 @@ free_virtio_hw(struct virtio_hw *hw) } static void -pci_dump_json_config(struct virtio_dev *dev, struct spdk_json_write_ctx *w) +pci_dump_json_info(struct virtio_dev *dev, struct spdk_json_write_ctx *w) { struct virtio_hw *hw = dev->ctx; struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr((struct spdk_pci_device *)hw->pci_dev); @@ -126,6 +126,19 @@ pci_dump_json_config(struct virtio_dev *dev, struct spdk_json_write_ctx *w) spdk_json_write_string(w, addr); } +static void +pci_write_json_config(struct virtio_dev *dev, struct spdk_json_write_ctx *w) +{ + struct virtio_hw *hw = dev->ctx; + struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(hw->pci_dev); + char addr[32]; + + spdk_pci_addr_fmt(addr, sizeof(addr), &pci_addr); + + spdk_json_write_named_string(w, "trtype", "pci"); + spdk_json_write_named_string(w, "traddr", addr); +} + static inline void io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi) { @@ -314,7 +327,8 @@ static const struct virtio_dev_ops modern_ops = { .setup_queue = modern_setup_queue, .del_queue = modern_del_queue, .notify_queue = modern_notify_queue, - .dump_json_config = pci_dump_json_config, + .dump_json_info = pci_dump_json_info, + .write_json_config = pci_write_json_config, }; static void * diff --git a/lib/virtio/virtio_user.c b/lib/virtio/virtio_user.c index 35fbf7b4e..064a4131e 100644 --- a/lib/virtio/virtio_user.c +++ b/lib/virtio/virtio_user.c @@ -401,7 +401,7 @@ virtio_user_destroy(struct virtio_dev *vdev) } static void -virtio_user_dump_json_config(struct virtio_dev *vdev, struct spdk_json_write_ctx *w) +virtio_user_dump_json_info(struct virtio_dev *vdev, struct spdk_json_write_ctx *w) { struct virtio_user_dev *dev = vdev->ctx; @@ -412,6 +412,17 @@ virtio_user_dump_json_config(struct virtio_dev *vdev, struct spdk_json_write_ctx spdk_json_write_string(w, dev->path); } +static void +virtio_user_write_json_config(struct virtio_dev *vdev, struct spdk_json_write_ctx *w) +{ + struct virtio_user_dev *dev = vdev->ctx; + + spdk_json_write_named_string(w, "trtype", "user"); + spdk_json_write_named_string(w, "traddr", dev->path); + spdk_json_write_named_uint32(w, "vq_count", vdev->max_queues - vdev->fixed_queues_num); + spdk_json_write_named_uint32(w, "vq_size", virtio_dev_backend_ops(vdev)->get_queue_num(vdev, 0)); +} + static const struct virtio_dev_ops virtio_user_ops = { .read_dev_cfg = virtio_user_read_dev_config, .write_dev_cfg = virtio_user_write_dev_config, @@ -424,7 +435,8 @@ static const struct virtio_dev_ops virtio_user_ops = { .setup_queue = virtio_user_setup_queue, .del_queue = virtio_user_del_queue, .notify_queue = virtio_user_notify_queue, - .dump_json_config = virtio_user_dump_json_config, + .dump_json_info = virtio_user_dump_json_info, + .write_json_config = virtio_user_write_json_config, }; int diff --git a/scripts/rpc.py b/scripts/rpc.py index f6871ea86..9400a3a6c 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -889,6 +889,24 @@ if __name__ == "__main__": p.add_argument('ctrlr', help='controller name') p.set_defaults(func=remove_vhost_controller) + @call_cmd + def construct_virtio_dev(args): + print_dict(rpc.vhost.construct_virtio_dev(args.client, args)) + + p = subparsers.add_parser('construct_virtio_dev', help="""Construct new virtio device using provided + transport type and device type. In case of SCSI device type this implies scan and add bdevs offered by + remote side. Result is array of added bdevs.""") + p.add_argument('name', help="Use this name as base for new created bdevs") + p.add_argument('-t', '--trtype', + help='Virtio target transport type: pci or user', required=True) + p.add_argument('-a', '--traddr', + help='Transport type specific target address: e.g. UNIX domain socket path or BDF', required=True) + p.add_argument('-d', '--dev-type', + help='Device type: blk or scsi', required=True) + p.add_argument('--vq-count', help='Number of virtual queues to be used.', type=int) + p.add_argument('--vq-size', help='Size of each queue', type=int) + p.set_defaults(func=construct_virtio_dev) + @call_cmd def construct_virtio_user_scsi_bdev(args): print_dict(rpc.vhost.construct_virtio_user_scsi_bdev(args.client, args)) diff --git a/scripts/rpc/vhost.py b/scripts/rpc/vhost.py index d03d88797..4577da689 100755 --- a/scripts/rpc/vhost.py +++ b/scripts/rpc/vhost.py @@ -54,6 +54,20 @@ def remove_vhost_controller(client, args): return client.call('remove_vhost_controller', params) +def construct_virtio_dev(client, args): + params = { + 'name': args.name, + 'trtype': args.trtype, + 'traddr': args.traddr, + 'dev_type': args.dev_type + } + if args.vq_count: + params['vq_count'] = args.vq_count + if args.vq_size: + params['vq_size'] = args.vq_size + return client.call('construct_virtio_dev', params) + + def construct_virtio_user_scsi_bdev(client, args): params = { 'path': args.path,