From f81888b2f9e09ee881dbd220e2ec9ed079e418bc Mon Sep 17 00:00:00 2001 From: GangCao Date: Fri, 7 Oct 2016 20:01:54 -0400 Subject: [PATCH] nvme: add PCI BDF in spdk_nvme_ctrlr to check whether same ctrlr Change-Id: Ic8eb395bbfcc688e9c999a6d0026b70c24d386e3 Signed-off-by: GangCao --- include/spdk/env.h | 9 +++++ lib/env/pci.c | 9 +++++ lib/nvme/nvme.c | 7 ++-- lib/nvme/nvme_ctrlr.c | 6 ++++ lib/nvme/nvme_internal.h | 3 ++ test/lib/nvme/unit/nvme_c/nvme_ut.c | 6 ++++ .../nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c | 34 +++++++++++++++++++ .../nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c | 6 ++++ 8 files changed, 77 insertions(+), 3 deletions(-) diff --git a/include/spdk/env.h b/include/spdk/env.h index 71f2a453f..656f5a12c 100644 --- a/include/spdk/env.h +++ b/include/spdk/env.h @@ -152,6 +152,13 @@ enum spdk_pci_device_type { SPDK_PCI_DEVICE_IOAT, }; +struct spdk_pci_addr { + uint16_t domain; + uint8_t bus; + uint8_t dev; + uint8_t func; +}; + typedef int (*spdk_pci_enum_cb)(void *enum_ctx, struct spdk_pci_device *pci_dev); int spdk_pci_enumerate(enum spdk_pci_device_type type, @@ -182,6 +189,8 @@ int spdk_pci_device_cfg_write16(struct spdk_pci_device *dev, uint16_t value, uin int spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value, uint32_t offset); int spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value, uint32_t offset); +bool spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr); + #ifdef __cplusplus } #endif diff --git a/lib/env/pci.c b/lib/env/pci.c index f0cd0e6aa..b9cd82cde 100644 --- a/lib/env/pci.c +++ b/lib/env/pci.c @@ -385,6 +385,15 @@ spdk_pci_device_get_serial_number(struct spdk_pci_device *dev, char *sn, size_t return -1; } +bool +spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr) +{ + return ((spdk_pci_device_get_domain(dev) == addr->domain) && + (spdk_pci_device_get_bus(dev) == addr->bus) && + (spdk_pci_device_get_dev(dev) == addr->dev) && + (spdk_pci_device_get_func(dev) == addr->func)); +} + #ifdef __linux__ int spdk_pci_device_claim(struct spdk_pci_device *dev) diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c index eea09e959..0807c947c 100644 --- a/lib/nvme/nvme.c +++ b/lib/nvme/nvme.c @@ -243,10 +243,11 @@ nvme_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) /* Verify that this controller is not already attached */ TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) { - /* NOTE: This assumes that the PCI abstraction layer will use the same device handle - * across enumerations; we could compare by BDF instead if this is not true. + /* NOTE: In the case like multi-process environment where the device handle is + * different per each process, we compare by BDF to determine whether it is the + * same controller. */ - if (pci_dev == ctrlr->devhandle) { + if (spdk_pci_device_compare_addr(pci_dev, &ctrlr->pci_addr)) { return 0; } } diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index e1b6f4d0d..f986a34fe 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -1140,6 +1140,12 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr, void *devhandle) pthread_mutex_init_recursive(&ctrlr->ctrlr_lock); + /* Save the PCI address */ + ctrlr->pci_addr.domain = spdk_pci_device_get_domain(devhandle); + ctrlr->pci_addr.bus = spdk_pci_device_get_bus(devhandle); + ctrlr->pci_addr.dev = spdk_pci_device_get_dev(devhandle); + ctrlr->pci_addr.func = spdk_pci_device_get_func(devhandle); + return 0; } diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index 4484abc77..bc6cc90cb 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -459,6 +459,9 @@ struct spdk_nvme_ctrlr { uint64_t cmb_size; /** Current offset of controller memory buffer */ uint64_t cmb_current_offset; + + /** PCI address including domain, bus, device and function */ + struct spdk_pci_addr pci_addr; }; struct nvme_driver { diff --git a/test/lib/nvme/unit/nvme_c/nvme_ut.c b/test/lib/nvme/unit/nvme_c/nvme_ut.c index 1f8d3022c..bcd45bf4c 100644 --- a/test/lib/nvme/unit/nvme_c/nvme_ut.c +++ b/test/lib/nvme/unit/nvme_c/nvme_ut.c @@ -76,6 +76,12 @@ spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts) memset(opts, 0, sizeof(*opts)); } +bool +spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr) +{ + return true; +} + static void test_opc_data_transfer(void) { diff --git a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c index f3b4b6907..d7672f7ec 100644 --- a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c +++ b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c @@ -51,6 +51,10 @@ static uint16_t g_pci_vendor_id; static uint16_t g_pci_device_id; static uint16_t g_pci_subvendor_id; static uint16_t g_pci_subdevice_id; +static uint16_t g_pci_domain; +static uint8_t g_pci_bus; +static uint8_t g_pci_dev; +static uint8_t g_pci_func; uint64_t g_ut_tsc = 0; struct spdk_nvme_registers g_ut_nvme_regs = {}; @@ -112,6 +116,36 @@ spdk_pci_device_get_subdevice_id(struct spdk_pci_device *dev) return g_pci_subdevice_id; } +uint16_t +spdk_pci_device_get_domain(struct spdk_pci_device *dev) +{ + return g_pci_domain; +} + +uint8_t +spdk_pci_device_get_bus(struct spdk_pci_device *dev) +{ + return g_pci_bus; +} + +uint8_t +spdk_pci_device_get_dev(struct spdk_pci_device *dev) +{ + return g_pci_dev; +} + +uint8_t +spdk_pci_device_get_func(struct spdk_pci_device *dev) +{ + return g_pci_func; +} + +bool +spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr) +{ + return true; +} + int nvme_qpair_construct(struct spdk_nvme_qpair *qpair, uint16_t id, uint16_t num_entries, uint16_t num_trackers, struct spdk_nvme_ctrlr *ctrlr) diff --git a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c index 3f3e313b6..d70cc41e3 100644 --- a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c +++ b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c @@ -80,6 +80,12 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr) return 0; } +bool +spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr) +{ + return true; +} + void spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts) {