From 2969c5ab30589c0e63177bb343cf20162371004a Mon Sep 17 00:00:00 2001 From: Jin Yu Date: Wed, 19 Aug 2020 18:02:07 +0800 Subject: [PATCH] vhost-blk: recover ring base when reconnect This patch is for packed ring and recover the ring base when vhost target reconnect to QEMU. Change-Id: I73f791b4a55adf9834112afd7dd7bb26c75a135d Signed-off-by: Jin Yu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/4128 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Changpeng Liu --- lib/vhost/vhost.c | 15 +++++++++++++++ lib/vhost/vhost_internal.h | 2 ++ lib/vhost/vhost_rpc.c | 4 ++++ scripts/rpc.py | 4 +++- scripts/rpc/vhost.py | 5 ++++- test/unit/lib/vhost/vhost.c/vhost_ut.c | 2 ++ 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index 1596e4ab8..b3aee7259 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -42,6 +42,8 @@ #include "spdk/vhost.h" #include "vhost_internal.h" +bool g_packed_ring_recovery = false; + static struct spdk_cpuset g_vhost_core_mask; /* Path to folder where character device will be created. Can be set by user. */ @@ -1340,6 +1342,19 @@ vhost_start_device_cb(int vid) } if (packed_ring) { + /* Use the inflight mem to restore the last_avail_idx and last_used_idx. + * When the vring format is packed, there is no used_idx in the + * used ring, so VM can't resend the used_idx to VHOST when reconnect. + * QEMU version 5.2.0 supports the packed inflight before that it only + * supports split ring inflight because it doesn't send negotiated features + * before get inflight fd. Users can use RPC to enable this function. + */ + if (spdk_unlikely(g_packed_ring_recovery)) { + rte_vhost_get_vring_base_from_inflight(vsession->vid, i, + &q->last_avail_idx, + &q->last_used_idx); + } + /* Packed virtqueues support up to 2^15 entries each * so left one bit can be used as wrap counter. */ diff --git a/lib/vhost/vhost_internal.h b/lib/vhost/vhost_internal.h index 9172edbbf..dee1a28e9 100644 --- a/lib/vhost/vhost_internal.h +++ b/lib/vhost/vhost_internal.h @@ -45,6 +45,8 @@ #include "spdk/rpc.h" #include "spdk/config.h" +extern bool g_packed_ring_recovery; + #define SPDK_VHOST_MAX_VQUEUES 256 #define SPDK_VHOST_MAX_VQ_SIZE 1024 diff --git a/lib/vhost/vhost_rpc.c b/lib/vhost/vhost_rpc.c index fd8c40f74..b1555f2fb 100644 --- a/lib/vhost/vhost_rpc.c +++ b/lib/vhost/vhost_rpc.c @@ -237,6 +237,7 @@ struct rpc_vhost_blk_ctrlr { char *cpumask; bool readonly; bool packed_ring; + bool packed_ring_recovery; }; static const struct spdk_json_object_decoder rpc_construct_vhost_blk_ctrlr[] = { @@ -245,6 +246,7 @@ static const struct spdk_json_object_decoder rpc_construct_vhost_blk_ctrlr[] = { {"cpumask", offsetof(struct rpc_vhost_blk_ctrlr, cpumask), spdk_json_decode_string, true}, {"readonly", offsetof(struct rpc_vhost_blk_ctrlr, readonly), spdk_json_decode_bool, true}, {"packed_ring", offsetof(struct rpc_vhost_blk_ctrlr, packed_ring), spdk_json_decode_bool, true}, + {"packed_ring_recovery", offsetof(struct rpc_vhost_blk_ctrlr, packed_ring_recovery), spdk_json_decode_bool, true}, }; static void @@ -270,6 +272,8 @@ rpc_vhost_create_blk_controller(struct spdk_jsonrpc_request *request, goto invalid; } + g_packed_ring_recovery = req.packed_ring_recovery; + rc = spdk_vhost_blk_construct(req.ctrlr, req.cpumask, req.dev_name, req.readonly, req.packed_ring); if (rc < 0) { diff --git a/scripts/rpc.py b/scripts/rpc.py index ec9895594..313eaad15 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2202,7 +2202,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse dev_name=args.dev_name, cpumask=args.cpumask, readonly=args.readonly, - packed_ring=args.packed_ring) + packed_ring=args.packed_ring, + packed_ring_recovery=args.packed_ring_recovery) p = subparsers.add_parser('vhost_create_blk_controller', aliases=['construct_vhost_blk_controller'], @@ -2212,6 +2213,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p.add_argument('--cpumask', help='cpu mask for this controller') p.add_argument("-r", "--readonly", action='store_true', help='Set controller as read-only') p.add_argument("-p", "--packed_ring", action='store_true', help='Set controller as packed ring supported') + p.add_argument("-l", "--packed_ring_recovery", action='store_true', help='Enable packed ring live reocvery') p.set_defaults(func=vhost_create_blk_controller) def vhost_get_controllers(args): diff --git a/scripts/rpc/vhost.py b/scripts/rpc/vhost.py index f34a6f188..8dc17d07c 100644 --- a/scripts/rpc/vhost.py +++ b/scripts/rpc/vhost.py @@ -63,7 +63,7 @@ def vhost_scsi_controller_remove_target(client, ctrlr, scsi_target_num): @deprecated_alias('construct_vhost_blk_controller') -def vhost_create_blk_controller(client, ctrlr, dev_name, cpumask=None, readonly=None, packed_ring=None): +def vhost_create_blk_controller(client, ctrlr, dev_name, cpumask=None, readonly=None, packed_ring=None, packed_ring_recovery=None): """Create vhost BLK controller. Args: ctrlr: controller name @@ -71,6 +71,7 @@ def vhost_create_blk_controller(client, ctrlr, dev_name, cpumask=None, readonly= cpumask: cpu mask for this controller readonly: set controller as read-only packed_ring: support controller packed_ring + packed_ring_recovery: enable packed ring live recovery """ params = { 'ctrlr': ctrlr, @@ -82,6 +83,8 @@ def vhost_create_blk_controller(client, ctrlr, dev_name, cpumask=None, readonly= params['readonly'] = readonly if packed_ring: params['packed_ring'] = packed_ring + if packed_ring_recovery: + params['packed_ring_recovery'] = packed_ring_recovery return client.call('vhost_create_blk_controller', params) diff --git a/test/unit/lib/vhost/vhost.c/vhost_ut.c b/test/unit/lib/vhost/vhost.c/vhost_ut.c index 9a1c30a9c..79a6d204e 100644 --- a/test/unit/lib/vhost/vhost.c/vhost_ut.c +++ b/test/unit/lib/vhost/vhost.c/vhost_ut.c @@ -86,6 +86,8 @@ DEFINE_STUB(vhost_get_negotiated_features, int, (int vid, uint64_t *negotiated_features), 0); DEFINE_STUB(rte_vhost_get_vhost_ring_inflight, int, (int vid, uint16_t vring_idx, struct rte_vhost_ring_inflight *vring), 0); +DEFINE_STUB(rte_vhost_get_vring_base_from_inflight, int, + (int vid, uint16_t queue_id, uint16_t *last_avail_idx, uint16_t *last_used_idx), 0); DEFINE_STUB(vhost_get_mem_table, int, (int vid, struct rte_vhost_memory **mem), 0); void *