ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme/vfio-user: use API to setup BAR0 doorbells

Browse Source 
We can use lib/vfio-user API to setup BAR0 doorbells,
existing implementation is redundant.

Change-Id: Ib880d167c84c6b8482bf1a35559a34c939f6a02d
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12211
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: John Levon <levon@movementarian.org>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
This commit is contained in:
Changpeng Liu 2022-04-11 10:13:33 +08:00 committed by Tomasz Zawadzki
parent 04b16168cf
commit c47b7b0276
1 changed files with 13 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
lib/nvme/nvme_vfio_user.c
View File

@ -49,7 +49,6 @@ struct nvme_vfio_ctrlr {
struct nvme_pcie_ctrlr pctrlr; struct nvme_pcie_ctrlr pctrlr;
volatile uint32_t *doorbell_base; volatile uint32_t *doorbell_base;
int bar0_fd;
struct vfio_device *dev; struct vfio_device *dev;
}; };
@ -148,44 +147,20 @@ nvme_vfio_ctrlr_set_aqa(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_aqa
aqa->raw); aqa->raw);
} }
/* Instead of using path as the bar0 file descriptor, we can also use
* SPARSE MMAP to get the doorbell mmaped address.
*/
static int static int
nvme_vfio_setup_bar0(struct nvme_vfio_ctrlr *vctrlr, const char *path) nvme_vfio_setup_bar0(struct nvme_vfio_ctrlr *vctrlr)
{ {
volatile uint32_t *doorbell; void *doorbell;
int fd;
fd = open(path, O_RDWR); doorbell = spdk_vfio_user_get_bar_addr(vctrlr->dev, 0, 0x1000, 0x1000);
if (fd < 0) { if (!doorbell) {
SPDK_ERRLOG("Failed to open file %s\n", path); return -EINVAL;
return fd;
} }
doorbell = mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0x1000); vctrlr->doorbell_base = (volatile uint32_t *)doorbell;
if (doorbell == MAP_FAILED) {
SPDK_ERRLOG("Failed to mmap file %s\n", path);
close(fd);
return -EFAULT;
}
vctrlr->bar0_fd = fd;
vctrlr->doorbell_base = doorbell;
return 0; return 0;
} }
static void
nvme_vfio_bar0_destruct(struct nvme_vfio_ctrlr *vctrlr)
{
if (vctrlr->doorbell_base) {
munmap((void *)vctrlr->doorbell_base, 0x1000);
}
close(vctrlr->bar0_fd);
}
static struct spdk_nvme_ctrlr * static struct spdk_nvme_ctrlr *
nvme_vfio_ctrlr_construct(const struct spdk_nvme_transport_id *trid, nvme_vfio_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
const struct spdk_nvme_ctrlr_opts *opts, const struct spdk_nvme_ctrlr_opts *opts,
@ -197,40 +172,30 @@ static struct spdk_nvme_ctrlr *
union spdk_nvme_cap_register cap; union spdk_nvme_cap_register cap;
int ret; int ret;
char ctrlr_path[PATH_MAX]; char ctrlr_path[PATH_MAX];
char ctrlr_bar0[PATH_MAX];
snprintf(ctrlr_path, sizeof(ctrlr_path), "%s/cntrl", trid->traddr); snprintf(ctrlr_path, sizeof(ctrlr_path), "%s/cntrl", trid->traddr);
snprintf(ctrlr_bar0, sizeof(ctrlr_bar0), "%s/bar0", trid->traddr);
ret = access(ctrlr_path, F_OK); ret = access(ctrlr_path, F_OK);
if (ret != 0) { if (ret != 0) {
SPDK_ERRLOG("Access path %s failed\n", ctrlr_path); SPDK_ERRLOG("Access path %s failed\n", ctrlr_path);
return NULL; return NULL;
} }
ret = access(ctrlr_bar0, F_OK);
if (ret != 0) {
SPDK_ERRLOG("Access path %s failed\n", ctrlr_bar0);
return NULL;
}
vctrlr = calloc(1, sizeof(*vctrlr)); vctrlr = calloc(1, sizeof(*vctrlr));
if (!vctrlr) { if (!vctrlr) {
return NULL; return NULL;
} }
ret = nvme_vfio_setup_bar0(vctrlr, ctrlr_bar0); vctrlr->dev = spdk_vfio_user_setup(ctrlr_path);
if (ret != 0) { if (!vctrlr->dev) {
SPDK_ERRLOG("Error to setup vfio device\n");
free(vctrlr); free(vctrlr);
return NULL; return NULL;
} }
vctrlr->dev = spdk_vfio_user_setup(ctrlr_path); ret = nvme_vfio_setup_bar0(vctrlr);
if (!vctrlr->dev) { if (ret != 0) {
SPDK_ERRLOG("Error to setup vfio device\n"); SPDK_ERRLOG("Error to get device BAR0\n");
nvme_vfio_bar0_destruct(vctrlr); goto exit;
free(vctrlr);
return NULL;
} }
pctrlr = &vctrlr->pctrlr; pctrlr = &vctrlr->pctrlr;
@ -285,7 +250,6 @@ static struct spdk_nvme_ctrlr *
return &pctrlr->ctrlr; return &pctrlr->ctrlr;
exit: exit:
nvme_vfio_bar0_destruct(vctrlr);
spdk_vfio_user_release(vctrlr->dev); spdk_vfio_user_release(vctrlr->dev);
free(vctrlr); free(vctrlr);
return NULL; return NULL;
@ -354,7 +318,6 @@ nvme_vfio_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
nvme_ctrlr_free_processes(ctrlr); nvme_ctrlr_free_processes(ctrlr);
nvme_vfio_bar0_destruct(vctrlr);
spdk_vfio_user_release(vctrlr->dev); spdk_vfio_user_release(vctrlr->dev);
free(vctrlr); free(vctrlr);