diff --git a/lib/env_dpdk/vtophys.c b/lib/env_dpdk/vtophys.c index cf654f0ee..7d0a5e1ce 100644 --- a/lib/env_dpdk/vtophys.c +++ b/lib/env_dpdk/vtophys.c @@ -70,13 +70,13 @@ /* Max value for a 16-bit ref count. */ #define VTOPHYS_MAX_REF_COUNT (0xFFFF) -/* Physical address of a single 2MB page. */ +/* Translation of a single 2MB page. */ struct map_2mb { - uint64_t paddr_2mb; + uint64_t translation_2mb; }; /* Second-level map table indexed by bits [21..29] of the virtual address. - * Each entry contains the 2MB physical address or SPDK_VTOPHYS_ERROR for entries that haven't + * Each entry contains the address translation or SPDK_VTOPHYS_ERROR for entries that haven't * been retrieved yet. */ struct map_1gb { @@ -91,33 +91,38 @@ struct map_128tb { struct map_1gb *map[1ULL << (SHIFT_128TB - SHIFT_1GB + 1)]; }; -static struct map_128tb vtophys_map_128tb = {}; -static pthread_mutex_t vtophys_mutex = PTHREAD_MUTEX_INITIALIZER; +/* Page-granularity memory address translation */ +struct spdk_mem_map { + struct map_128tb map_128tb; + pthread_mutex_t mutex; +}; + +static struct spdk_mem_map g_vtophys_map = {{}, PTHREAD_MUTEX_INITIALIZER}; static struct map_1gb * -vtophys_get_map_1gb(uint64_t vfn_2mb) +spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb) { struct map_1gb *map_1gb; uint64_t idx_128tb = MAP_128TB_IDX(vfn_2mb); - map_1gb = vtophys_map_128tb.map[idx_128tb]; + map_1gb = map->map_128tb.map[idx_128tb]; if (!map_1gb) { - pthread_mutex_lock(&vtophys_mutex); + pthread_mutex_lock(&map->mutex); /* Recheck to make sure nobody else got the mutex first. */ - map_1gb = vtophys_map_128tb.map[idx_128tb]; + map_1gb = map->map_128tb.map[idx_128tb]; if (!map_1gb) { map_1gb = malloc(sizeof(struct map_1gb)); if (map_1gb) { /* initialize all entries to all 0xFF (SPDK_VTOPHYS_ERROR) */ memset(map_1gb->map, 0xFF, sizeof(map_1gb->map)); memset(map_1gb->ref_count, 0, sizeof(map_1gb->ref_count)); - vtophys_map_128tb.map[idx_128tb] = map_1gb; + map->map_128tb.map[idx_128tb] = map_1gb; } } - pthread_mutex_unlock(&vtophys_mutex); + pthread_mutex_unlock(&map->mutex); if (!map_1gb) { #ifdef DEBUG @@ -130,6 +135,116 @@ vtophys_get_map_1gb(uint64_t vfn_2mb) return map_1gb; } +static void +spdk_mem_map_register(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size, uint64_t translation) +{ + uint64_t vfn_2mb; + struct map_1gb *map_1gb; + uint64_t idx_1gb; + struct map_2mb *map_2mb; + uint16_t *ref_count; + + /* For now, only 2 MB registrations are supported */ + assert(size == 2 * 1024 * 1024); + assert((vaddr & MASK_2MB) == 0); + + vfn_2mb = vaddr >> SHIFT_2MB; + + map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb); + if (!map_1gb) { +#ifdef DEBUG + fprintf(stderr, "could not get %p map\n", (void *)vaddr); +#endif + return; + } + + idx_1gb = MAP_1GB_IDX(vfn_2mb); + map_2mb = &map_1gb->map[idx_1gb]; + ref_count = &map_1gb->ref_count[idx_1gb]; + + if (*ref_count == VTOPHYS_MAX_REF_COUNT) { +#ifdef DEBUG + fprintf(stderr, "ref count for %p already at %d\n", + (void *)vaddr, VTOPHYS_MAX_REF_COUNT); +#endif + return; + } + + map_2mb->translation_2mb = translation; + + (*ref_count)++; +} + +static void +spdk_mem_map_unregister(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size) +{ + uint64_t vfn_2mb; + struct map_1gb *map_1gb; + uint64_t idx_1gb; + struct map_2mb *map_2mb; + uint16_t *ref_count; + + /* For now, only 2 MB registrations are supported */ + assert(size == 2 * 1024 * 1024); + assert((vaddr & MASK_2MB) == 0); + + vfn_2mb = vaddr >> SHIFT_2MB; + + map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb); + if (!map_1gb) { +#ifdef DEBUG + fprintf(stderr, "could not get %p map\n", (void *)vaddr); +#endif + return; + } + + idx_1gb = MAP_1GB_IDX(vfn_2mb); + map_2mb = &map_1gb->map[idx_1gb]; + ref_count = &map_1gb->ref_count[idx_1gb]; + + if (*ref_count == 0) { +#ifdef DEBUG + fprintf(stderr, "vaddr %p not registered\n", (void *)vaddr); +#endif + return; + } + + (*ref_count)--; + if (*ref_count == 0) { + map_2mb->translation_2mb = SPDK_VTOPHYS_ERROR; + } +} + +static uint64_t +spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr) +{ + const struct map_1gb *map_1gb; + const struct map_2mb *map_2mb; + uint64_t idx_128tb; + uint64_t idx_1gb; + uint64_t vfn_2mb; + + if (spdk_unlikely(vaddr & ~MASK_128TB)) { +#ifdef DEBUG + printf("invalid usermode virtual address %p\n", (void *)vaddr); +#endif + return SPDK_VTOPHYS_ERROR; + } + + vfn_2mb = vaddr >> SHIFT_2MB; + idx_128tb = MAP_128TB_IDX(vfn_2mb); + idx_1gb = MAP_1GB_IDX(vfn_2mb); + + map_1gb = map->map_128tb.map[idx_128tb]; + if (spdk_unlikely(!map_1gb)) { + return SPDK_VTOPHYS_ERROR; + } + + map_2mb = &map_1gb->map[idx_1gb]; + + return map_2mb->translation_2mb; +} + static uint64_t vtophys_get_dpdk_paddr(void *vaddr) { @@ -187,11 +302,6 @@ vtophys_get_paddr(uint64_t vaddr) static void _spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr) { - struct map_1gb *map_1gb; - uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb); - struct map_2mb *map_2mb; - uint16_t *ref_count; - if (paddr & MASK_2MB) { #ifdef DEBUG fprintf(stderr, "invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr); @@ -199,60 +309,13 @@ _spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr) return; } - map_1gb = vtophys_get_map_1gb(vfn_2mb); - if (!map_1gb) { -#ifdef DEBUG - fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb); -#endif - return; - } - - map_2mb = &map_1gb->map[idx_1gb]; - ref_count = &map_1gb->ref_count[idx_1gb]; - - if (*ref_count == VTOPHYS_MAX_REF_COUNT) { -#ifdef DEBUG - fprintf(stderr, "ref count for %p already at %d\n", - (void *)(vfn_2mb << SHIFT_2MB), VTOPHYS_MAX_REF_COUNT); -#endif - return; - } - - map_2mb->paddr_2mb = paddr; - - (*ref_count)++; + spdk_mem_map_register(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024, paddr); } static void _spdk_vtophys_unregister_one(uint64_t vfn_2mb) { - struct map_1gb *map_1gb; - uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb); - struct map_2mb *map_2mb; - uint16_t *ref_count; - - map_1gb = vtophys_get_map_1gb(vfn_2mb); - if (!map_1gb) { -#ifdef DEBUG - fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb); -#endif - return; - } - - map_2mb = &map_1gb->map[idx_1gb]; - ref_count = &map_1gb->ref_count[idx_1gb]; - - if (map_2mb->paddr_2mb == SPDK_VTOPHYS_ERROR || *ref_count == 0) { -#ifdef DEBUG - fprintf(stderr, "vaddr %p not registered\n", (void *)(vfn_2mb << SHIFT_2MB)); -#endif - return; - } - - (*ref_count)--; - if (*ref_count == 0) { - map_2mb->paddr_2mb = SPDK_VTOPHYS_ERROR; - } + spdk_mem_map_unregister(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024); } void @@ -347,32 +410,11 @@ spdk_vtophys_register_dpdk_mem(void) uint64_t spdk_vtophys(void *buf) { - struct map_1gb *map_1gb; - struct map_2mb *map_2mb; - uint64_t idx_128tb; - uint64_t idx_1gb; - uint64_t vaddr, vfn_2mb, paddr_2mb; + uint64_t vaddr, paddr_2mb; vaddr = (uint64_t)buf; - if (spdk_unlikely(vaddr & ~MASK_128TB)) { -#ifdef DEBUG - printf("invalid usermode virtual address %p\n", buf); -#endif - return SPDK_VTOPHYS_ERROR; - } - vfn_2mb = vaddr >> SHIFT_2MB; - idx_128tb = MAP_128TB_IDX(vfn_2mb); - idx_1gb = MAP_1GB_IDX(vfn_2mb); - - map_1gb = vtophys_map_128tb.map[idx_128tb]; - if (spdk_unlikely(!map_1gb)) { - return SPDK_VTOPHYS_ERROR; - } - - map_2mb = &map_1gb->map[idx_1gb]; - - paddr_2mb = map_2mb->paddr_2mb; + paddr_2mb = spdk_mem_map_translate(&g_vtophys_map, vaddr); /* * SPDK_VTOPHYS_ERROR has all bits set, so if the lookup returned SPDK_VTOPHYS_ERROR,