diff --git a/lib/env_dpdk/env_internal.h b/lib/env_dpdk/env_internal.h index 72bebcc36..282600a9e 100644 --- a/lib/env_dpdk/env_internal.h +++ b/lib/env_dpdk/env_internal.h @@ -53,11 +53,11 @@ extern struct rte_pci_bus rte_pci_bus; #endif #include -/* x86-64 userspace virtual addresses use only the low 47 bits [0..46], - * which is enough to cover 128 TB. +/* x86-64 and ARM userspace virtual addresses use only the low 48 bits [0..47], + * which is enough to cover 256 TB. */ -#define SHIFT_128TB 47 /* (1 << 47) == 128 TB */ -#define MASK_128TB ((1ULL << SHIFT_128TB) - 1) +#define SHIFT_256TB 48 /* (1 << 48) == 256 TB */ +#define MASK_256TB ((1ULL << SHIFT_256TB) - 1) #define SHIFT_1GB 30 /* (1 << 30) == 1 GB */ #define MASK_1GB ((1ULL << SHIFT_1GB) - 1) diff --git a/lib/env_dpdk/memory.c b/lib/env_dpdk/memory.c index 83351abed..aabec7b3d 100644 --- a/lib/env_dpdk/memory.c +++ b/lib/env_dpdk/memory.c @@ -54,7 +54,7 @@ #define FN_2MB_TO_4KB(fn) (fn << (SHIFT_2MB - SHIFT_4KB)) #define FN_4KB_TO_2MB(fn) (fn >> (SHIFT_2MB - SHIFT_4KB)) -#define MAP_128TB_IDX(vfn_2mb) ((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB)) +#define MAP_256TB_IDX(vfn_2mb) ((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB)) #define MAP_1GB_IDX(vfn_2mb) ((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB + 1)) - 1)) /* Translation of a single 2MB page. */ @@ -73,13 +73,13 @@ struct map_1gb { /* Top-level map table indexed by bits [30..46] of the virtual address. * Each entry points to a second-level map table or NULL. */ -struct map_128tb { - struct map_1gb *map[1ULL << (SHIFT_128TB - SHIFT_1GB + 1)]; +struct map_256tb { + struct map_1gb *map[1ULL << (SHIFT_256TB - SHIFT_1GB + 1)]; }; /* Page-granularity memory address translation */ struct spdk_mem_map { - struct map_128tb map_128tb; + struct map_256tb map_256tb; pthread_mutex_t mutex; uint64_t default_translation; spdk_mem_map_notify_cb notify_cb; @@ -98,7 +98,7 @@ static pthread_mutex_t g_spdk_mem_map_mutex = PTHREAD_MUTEX_INITIALIZER; static void spdk_mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_action action) { - size_t idx_128tb; + size_t idx_256tb; uint64_t contig_start = 0; uint64_t contig_end = 0; @@ -121,10 +121,10 @@ spdk_mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_acti /* Hold the memory registration map mutex so no new registrations can be added while we are looping. */ pthread_mutex_lock(&g_mem_reg_map->mutex); - for (idx_128tb = 0; - idx_128tb < sizeof(g_mem_reg_map->map_128tb.map) / sizeof(g_mem_reg_map->map_128tb.map[0]); - idx_128tb++) { - const struct map_1gb *map_1gb = g_mem_reg_map->map_128tb.map[idx_128tb]; + for (idx_256tb = 0; + idx_256tb < sizeof(g_mem_reg_map->map_256tb.map) / sizeof(g_mem_reg_map->map_256tb.map[0]); + idx_256tb++) { + const struct map_1gb *map_1gb = g_mem_reg_map->map_256tb.map[idx_256tb]; uint64_t idx_1gb; if (!map_1gb) { @@ -135,7 +135,7 @@ spdk_mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_acti for (idx_1gb = 0; idx_1gb < sizeof(map_1gb->map) / sizeof(map_1gb->map[0]); idx_1gb++) { if (map_1gb->map[idx_1gb].translation_2mb != 0) { /* Rebuild the virtual address from the indexes */ - uint64_t vaddr = (idx_128tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB); + uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB); if (contig_start == 0) { contig_start = vaddr; @@ -202,8 +202,8 @@ spdk_mem_map_free(struct spdk_mem_map **pmap) TAILQ_REMOVE(&g_spdk_mem_maps, map, tailq); pthread_mutex_unlock(&g_spdk_mem_map_mutex); - for (i = 0; i < sizeof(map->map_128tb.map) / sizeof(map->map_128tb.map[0]); i++) { - free(map->map_128tb.map[i]); + for (i = 0; i < sizeof(map->map_256tb.map) / sizeof(map->map_256tb.map[0]); i++) { + free(map->map_256tb.map[i]); } pthread_mutex_destroy(&map->mutex); @@ -220,7 +220,7 @@ spdk_mem_register(void *vaddr, size_t len) void *seg_vaddr; size_t seg_len; - if ((uintptr_t)vaddr & ~MASK_128TB) { + if ((uintptr_t)vaddr & ~MASK_256TB) { DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr); return -EINVAL; } @@ -286,7 +286,7 @@ spdk_mem_unregister(void *vaddr, size_t len) size_t seg_len; uint64_t ref_count; - if ((uintptr_t)vaddr & ~MASK_128TB) { + if ((uintptr_t)vaddr & ~MASK_256TB) { DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr); return -EINVAL; } @@ -357,16 +357,16 @@ static struct map_1gb * spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb) { struct map_1gb *map_1gb; - uint64_t idx_128tb = MAP_128TB_IDX(vfn_2mb); + uint64_t idx_256tb = MAP_256TB_IDX(vfn_2mb); size_t i; - map_1gb = map->map_128tb.map[idx_128tb]; + map_1gb = map->map_256tb.map[idx_256tb]; if (!map_1gb) { pthread_mutex_lock(&map->mutex); /* Recheck to make sure nobody else got the mutex first. */ - map_1gb = map->map_128tb.map[idx_128tb]; + map_1gb = map->map_256tb.map[idx_256tb]; if (!map_1gb) { map_1gb = malloc(sizeof(struct map_1gb)); if (map_1gb) { @@ -374,7 +374,7 @@ spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb) for (i = 0; i < SPDK_COUNTOF(map_1gb->map); i++) { map_1gb->map[i].translation_2mb = map->default_translation; } - map->map_128tb.map[idx_128tb] = map_1gb; + map->map_256tb.map[idx_256tb] = map_1gb; } } @@ -399,7 +399,7 @@ spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t struct map_2mb *map_2mb; /* For now, only 2 MB-aligned registrations are supported */ - if ((uintptr_t)vaddr & ~MASK_128TB) { + if ((uintptr_t)vaddr & ~MASK_256TB) { DEBUG_PRINT("invalid usermode virtual address %lu\n", vaddr); return -EINVAL; } @@ -439,7 +439,7 @@ spdk_mem_map_clear_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_ struct map_2mb *map_2mb; /* For now, only 2 MB-aligned registrations are supported */ - if ((uintptr_t)vaddr & ~MASK_128TB) { + if ((uintptr_t)vaddr & ~MASK_256TB) { DEBUG_PRINT("invalid usermode virtual address %lu\n", vaddr); return -EINVAL; } @@ -475,20 +475,20 @@ spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr) { const struct map_1gb *map_1gb; const struct map_2mb *map_2mb; - uint64_t idx_128tb; + uint64_t idx_256tb; uint64_t idx_1gb; uint64_t vfn_2mb; - if (spdk_unlikely(vaddr & ~MASK_128TB)) { + if (spdk_unlikely(vaddr & ~MASK_256TB)) { DEBUG_PRINT("invalid usermode virtual address %p\n", (void *)vaddr); return map->default_translation; } vfn_2mb = vaddr >> SHIFT_2MB; - idx_128tb = MAP_128TB_IDX(vfn_2mb); + idx_256tb = MAP_256TB_IDX(vfn_2mb); idx_1gb = MAP_1GB_IDX(vfn_2mb); - map_1gb = map->map_128tb.map[idx_128tb]; + map_1gb = map->map_256tb.map[idx_256tb]; if (spdk_unlikely(!map_1gb)) { return map->default_translation; } diff --git a/lib/env_dpdk/vtophys.c b/lib/env_dpdk/vtophys.c index bf5fd014a..92aa256d1 100644 --- a/lib/env_dpdk/vtophys.c +++ b/lib/env_dpdk/vtophys.c @@ -248,7 +248,7 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, int rc = 0; uint64_t paddr; - if ((uintptr_t)vaddr & ~MASK_128TB) { + if ((uintptr_t)vaddr & ~MASK_256TB) { DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr); return -EINVAL; }