ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vtophys: begin refactoring into general memory map

Browse Source 
Add a top-level structure that can be reused for other kinds of memory
address translations.

Change-Id: I046f98b76b4e98087d90095d6e9dea5cd6ab7898
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2017-02-27 10:20:41 -07:00
parent 44ceff8c17
commit 2c43c9bcca
1 changed files with 130 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
lib/env_dpdk/vtophys.c
View File

@ -70,13 +70,13 @@
/* Max value for a 16-bit ref count. */
#define VTOPHYS_MAX_REF_COUNT (0xFFFF)
/* Physical address of a single 2MB page. */
/* Translation of a single 2MB page. */
struct map_2mb {
uint64_t paddr_2mb;
uint64_t translation_2mb;
};
/* Second-level map table indexed by bits [21..29] of the virtual address.
* Each entry contains the 2MB physical address or SPDK_VTOPHYS_ERROR for entries that haven't
* Each entry contains the address translation or SPDK_VTOPHYS_ERROR for entries that haven't
* been retrieved yet.
*/
struct map_1gb {
@ -91,33 +91,38 @@ struct map_128tb {
struct map_1gb *map[1ULL << (SHIFT_128TB - SHIFT_1GB + 1)];
};
static struct map_128tb vtophys_map_128tb = {};
static pthread_mutex_t vtophys_mutex = PTHREAD_MUTEX_INITIALIZER;
/* Page-granularity memory address translation */
struct spdk_mem_map {
struct map_128tb map_128tb;
pthread_mutex_t mutex;
};
static struct spdk_mem_map g_vtophys_map = {{}, PTHREAD_MUTEX_INITIALIZER};
static struct map_1gb *
vtophys_get_map_1gb(uint64_t vfn_2mb)
spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb)
{
struct map_1gb *map_1gb;
uint64_t idx_128tb = MAP_128TB_IDX(vfn_2mb);
map_1gb = vtophys_map_128tb.map[idx_128tb];
map_1gb = map->map_128tb.map[idx_128tb];
if (!map_1gb) {
pthread_mutex_lock(&vtophys_mutex);
pthread_mutex_lock(&map->mutex);
/* Recheck to make sure nobody else got the mutex first. */
map_1gb = vtophys_map_128tb.map[idx_128tb];
map_1gb = map->map_128tb.map[idx_128tb];
if (!map_1gb) {
map_1gb = malloc(sizeof(struct map_1gb));
if (map_1gb) {
/* initialize all entries to all 0xFF (SPDK_VTOPHYS_ERROR) */
memset(map_1gb->map, 0xFF, sizeof(map_1gb->map));
memset(map_1gb->ref_count, 0, sizeof(map_1gb->ref_count));
vtophys_map_128tb.map[idx_128tb] = map_1gb;
map->map_128tb.map[idx_128tb] = map_1gb;
}
}
pthread_mutex_unlock(&vtophys_mutex);
pthread_mutex_unlock(&map->mutex);
if (!map_1gb) {
#ifdef DEBUG
@ -130,6 +135,116 @@ vtophys_get_map_1gb(uint64_t vfn_2mb)
return map_1gb;
}
static void
spdk_mem_map_register(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size, uint64_t translation)
{
uint64_t vfn_2mb;
struct map_1gb *map_1gb;
uint64_t idx_1gb;
struct map_2mb *map_2mb;
uint16_t *ref_count;
/* For now, only 2 MB registrations are supported */
assert(size == 2 * 1024 * 1024);
assert((vaddr & MASK_2MB) == 0);
vfn_2mb = vaddr >> SHIFT_2MB;
map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
if (!map_1gb) {
#ifdef DEBUG
fprintf(stderr, "could not get %p map\n", (void *)vaddr);
#endif
return;
}
idx_1gb = MAP_1GB_IDX(vfn_2mb);
map_2mb = &map_1gb->map[idx_1gb];
ref_count = &map_1gb->ref_count[idx_1gb];
if (*ref_count == VTOPHYS_MAX_REF_COUNT) {
#ifdef DEBUG
fprintf(stderr, "ref count for %p already at %d\n",
(void *)vaddr, VTOPHYS_MAX_REF_COUNT);
#endif
return;
}
map_2mb->translation_2mb = translation;
(*ref_count)++;
}
static void
spdk_mem_map_unregister(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size)
{
uint64_t vfn_2mb;
struct map_1gb *map_1gb;
uint64_t idx_1gb;
struct map_2mb *map_2mb;
uint16_t *ref_count;
/* For now, only 2 MB registrations are supported */
assert(size == 2 * 1024 * 1024);
assert((vaddr & MASK_2MB) == 0);
vfn_2mb = vaddr >> SHIFT_2MB;
map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
if (!map_1gb) {
#ifdef DEBUG
fprintf(stderr, "could not get %p map\n", (void *)vaddr);
#endif
return;
}
idx_1gb = MAP_1GB_IDX(vfn_2mb);
map_2mb = &map_1gb->map[idx_1gb];
ref_count = &map_1gb->ref_count[idx_1gb];
if (*ref_count == 0) {
#ifdef DEBUG
fprintf(stderr, "vaddr %p not registered\n", (void *)vaddr);
#endif
return;
}
(*ref_count)--;
if (*ref_count == 0) {
map_2mb->translation_2mb = SPDK_VTOPHYS_ERROR;
}
}
static uint64_t
spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr)
{
const struct map_1gb *map_1gb;
const struct map_2mb *map_2mb;
uint64_t idx_128tb;
uint64_t idx_1gb;
uint64_t vfn_2mb;
if (spdk_unlikely(vaddr & ~MASK_128TB)) {
#ifdef DEBUG
printf("invalid usermode virtual address %p\n", (void *)vaddr);
#endif
return SPDK_VTOPHYS_ERROR;
}
vfn_2mb = vaddr >> SHIFT_2MB;
idx_128tb = MAP_128TB_IDX(vfn_2mb);
idx_1gb = MAP_1GB_IDX(vfn_2mb);
map_1gb = map->map_128tb.map[idx_128tb];
if (spdk_unlikely(!map_1gb)) {
return SPDK_VTOPHYS_ERROR;
}
map_2mb = &map_1gb->map[idx_1gb];
return map_2mb->translation_2mb;
}
static uint64_t
vtophys_get_dpdk_paddr(void *vaddr)
{
@ -187,11 +302,6 @@ vtophys_get_paddr(uint64_t vaddr)
static void
_spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr)
{
struct map_1gb *map_1gb;
uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb);
struct map_2mb *map_2mb;
uint16_t *ref_count;
if (paddr & MASK_2MB) {
#ifdef DEBUG
fprintf(stderr, "invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr);
@ -199,60 +309,13 @@ _spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr)
return;
}
map_1gb = vtophys_get_map_1gb(vfn_2mb);
if (!map_1gb) {
#ifdef DEBUG
fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb);
#endif
return;
}
map_2mb = &map_1gb->map[idx_1gb];
ref_count = &map_1gb->ref_count[idx_1gb];
if (*ref_count == VTOPHYS_MAX_REF_COUNT) {
#ifdef DEBUG
fprintf(stderr, "ref count for %p already at %d\n",
(void *)(vfn_2mb << SHIFT_2MB), VTOPHYS_MAX_REF_COUNT);
#endif
return;
}
map_2mb->paddr_2mb = paddr;
(*ref_count)++;
spdk_mem_map_register(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024, paddr);
}
static void
_spdk_vtophys_unregister_one(uint64_t vfn_2mb)
{
struct map_1gb *map_1gb;
uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb);
struct map_2mb *map_2mb;
uint16_t *ref_count;
map_1gb = vtophys_get_map_1gb(vfn_2mb);
if (!map_1gb) {
#ifdef DEBUG
fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb);
#endif
return;
}
map_2mb = &map_1gb->map[idx_1gb];
ref_count = &map_1gb->ref_count[idx_1gb];
if (map_2mb->paddr_2mb == SPDK_VTOPHYS_ERROR || *ref_count == 0) {
#ifdef DEBUG
fprintf(stderr, "vaddr %p not registered\n", (void *)(vfn_2mb << SHIFT_2MB));
#endif
return;
}
(*ref_count)--;
if (*ref_count == 0) {
map_2mb->paddr_2mb = SPDK_VTOPHYS_ERROR;
}
spdk_mem_map_unregister(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024);
}
void
@ -347,32 +410,11 @@ spdk_vtophys_register_dpdk_mem(void)
uint64_t
spdk_vtophys(void *buf)
{
struct map_1gb *map_1gb;
struct map_2mb *map_2mb;
uint64_t idx_128tb;
uint64_t idx_1gb;
uint64_t vaddr, vfn_2mb, paddr_2mb;
uint64_t vaddr, paddr_2mb;
vaddr = (uint64_t)buf;
if (spdk_unlikely(vaddr & ~MASK_128TB)) {
#ifdef DEBUG
printf("invalid usermode virtual address %p\n", buf);
#endif
return SPDK_VTOPHYS_ERROR;
}
vfn_2mb = vaddr >> SHIFT_2MB;
idx_128tb = MAP_128TB_IDX(vfn_2mb);
idx_1gb = MAP_1GB_IDX(vfn_2mb);
map_1gb = vtophys_map_128tb.map[idx_128tb];
if (spdk_unlikely(!map_1gb)) {
return SPDK_VTOPHYS_ERROR;
}
map_2mb = &map_1gb->map[idx_1gb];
paddr_2mb = map_2mb->paddr_2mb;
paddr_2mb = spdk_mem_map_translate(&g_vtophys_map, vaddr);
/*
* SPDK_VTOPHYS_ERROR has all bits set, so if the lookup returned SPDK_VTOPHYS_ERROR,