From 4db2c11cd9fcb74498276264ad29aabd582e74a7 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Tue, 27 Apr 2021 21:46:24 +0800 Subject: [PATCH] nvmf/vfio-user: add NVMe device state definition for migration We will use the NVMe device state data structure to save/restore a NVMe controller in source/destination VM. NVMe device migration region is defined as below: ---------------------------------------------------------------------- | nvme_migr_device_state | private controller data | queue pairs | BARs | ---------------------------------------------------------------------- Change-Id: Idc73976e1de7f6da2da58e71db86df8cbb0d314d Signed-off-by: Changpeng Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7626 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- lib/nvmf/vfio_user.c | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index ab294a831..b2b1a3cc8 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -87,6 +87,84 @@ enum nvmf_vfio_user_req_state { VFIO_USER_REQUEST_STATE_EXECUTING, }; +/* NVMe device state representation */ +struct nvme_migr_sq_state { + uint16_t sqid; + uint16_t cqid; + uint32_t head; + uint32_t size; + uint32_t reserved; + uint64_t dma_addr; +}; +SPDK_STATIC_ASSERT(sizeof(struct nvme_migr_sq_state) == 0x18, "Incorrect size"); + +struct nvme_migr_cq_state { + uint16_t cqid; + uint16_t phase; + uint32_t tail; + uint32_t size; + uint32_t iv; + uint32_t ien; + uint32_t reserved; + uint64_t dma_addr; +}; +SPDK_STATIC_ASSERT(sizeof(struct nvme_migr_cq_state) == 0x20, "Incorrect size"); + +/* The device state is in VFIO MIGRATION BAR(9) region, keep the device state page aligned. + * + * NVMe device migration region is defined as below: + * ---------------------------------------------------------------------- + * | nvme_migr_device_state | private controller data | queue pairs | BARs | + * ---------------------------------------------------------------------- + * + * Keep nvme_migr_device_state as a fixed 0x1000 length, all new added fields + * can use the reserved space at the end of the data structure. + */ +struct nvme_migr_device_state { + /* Magic value to validate migration data */ + uint32_t magic; + /* Version to check the data is same from source to destination */ + uint32_t version; + + /* The library uses this field to know how many fields in this + * structure are valid, starting at the beginning of this data + * structure. New added fields in future use `unused` memory + * spaces. + */ + uint32_t opts_size; + uint32_t reserved0; + + /* BARs information */ + uint64_t bar_offset[VFU_PCI_DEV_NUM_REGIONS]; + uint64_t bar_len[VFU_PCI_DEV_NUM_REGIONS]; + + /* Queue pair start offset, starting at the beginning of this + * data structure. + */ + uint64_t qp_offset; + uint64_t qp_len; + + /* Controller data structure */ + uint32_t num_io_queues; + uint32_t reserved1; + + uint16_t reserved2[3]; + uint16_t nr_aers; + uint16_t aer_cids[256]; + + /* Controller private data offset and length if exist, starting at + * the beginning of this data structure. + */ + uint64_t private_data_offset; + uint64_t private_data_len; + + /* Reserved memory space for new added fields, the + * field is always at the end of this data structure. + */ + uint8_t unused[3356]; +}; +SPDK_STATIC_ASSERT(sizeof(struct nvme_migr_device_state) == 0x1000, "Incorrect size"); + struct nvmf_vfio_user_req { struct spdk_nvmf_request req; struct spdk_nvme_cpl rsp;