vtophys: begin refactoring into general memory map

Add a top-level structure that can be reused for other kinds of memory
address translations.

Change-Id: I046f98b76b4e98087d90095d6e9dea5cd6ab7898
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>

lib/env_dpdk/vtophys.c

@@ -70,13 +70,13 @@
 /* Max value for a 16-bit ref count. */
 #define VTOPHYS_MAX_REF_COUNT (0xFFFF)
 
-/* Physical address of a single 2MB page. */
+/* Translation of a single 2MB page. */
 struct map_2mb {
-	uint64_t paddr_2mb;
+	uint64_t translation_2mb;
 };
 
 /* Second-level map table indexed by bits [21..29] of the virtual address.
- * Each entry contains the 2MB physical address or SPDK_VTOPHYS_ERROR for entries that haven't
+ * Each entry contains the address translation or SPDK_VTOPHYS_ERROR for entries that haven't
  * been retrieved yet.
  */
 struct map_1gb {
@@ -91,33 +91,38 @@ struct map_128tb {
 	struct map_1gb *map[1ULL << (SHIFT_128TB - SHIFT_1GB + 1)];
 };
 
-static struct map_128tb vtophys_map_128tb = {};
+/* Page-granularity memory address translation */
-static pthread_mutex_t vtophys_mutex = PTHREAD_MUTEX_INITIALIZER;
+struct spdk_mem_map {
+	struct map_128tb map_128tb;
+	pthread_mutex_t mutex;
+};
+
+static struct spdk_mem_map g_vtophys_map = {{}, PTHREAD_MUTEX_INITIALIZER};
 
 static struct map_1gb *
-vtophys_get_map_1gb(uint64_t vfn_2mb)
+spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb)
 {
 	struct map_1gb *map_1gb;
 	uint64_t idx_128tb = MAP_128TB_IDX(vfn_2mb);
 
-	map_1gb = vtophys_map_128tb.map[idx_128tb];
+	map_1gb = map->map_128tb.map[idx_128tb];
 
 	if (!map_1gb) {
-		pthread_mutex_lock(&vtophys_mutex);
+		pthread_mutex_lock(&map->mutex);
 
 		/* Recheck to make sure nobody else got the mutex first. */
-		map_1gb = vtophys_map_128tb.map[idx_128tb];
+		map_1gb = map->map_128tb.map[idx_128tb];
 		if (!map_1gb) {
 			map_1gb = malloc(sizeof(struct map_1gb));
 			if (map_1gb) {
 				/* initialize all entries to all 0xFF (SPDK_VTOPHYS_ERROR) */
 				memset(map_1gb->map, 0xFF, sizeof(map_1gb->map));
 				memset(map_1gb->ref_count, 0, sizeof(map_1gb->ref_count));
-				vtophys_map_128tb.map[idx_128tb] = map_1gb;
+				map->map_128tb.map[idx_128tb] = map_1gb;
 			}
 		}
 
-		pthread_mutex_unlock(&vtophys_mutex);
+		pthread_mutex_unlock(&map->mutex);
 
 		if (!map_1gb) {
 #ifdef DEBUG
@@ -130,6 +135,116 @@ vtophys_get_map_1gb(uint64_t vfn_2mb)
 	return map_1gb;
 }
 
+static void
+spdk_mem_map_register(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size, uint64_t translation)
+{
+	uint64_t vfn_2mb;
+	struct map_1gb *map_1gb;
+	uint64_t idx_1gb;
+	struct map_2mb *map_2mb;
+	uint16_t *ref_count;
+
+	/* For now, only 2 MB registrations are supported */
+	assert(size == 2 * 1024 * 1024);
+	assert((vaddr & MASK_2MB) == 0);
+
+	vfn_2mb = vaddr >> SHIFT_2MB;
+
+	map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
+	if (!map_1gb) {
+#ifdef DEBUG
+		fprintf(stderr, "could not get %p map\n", (void *)vaddr);
+#endif
+		return;
+	}
+
+	idx_1gb = MAP_1GB_IDX(vfn_2mb);
+	map_2mb = &map_1gb->map[idx_1gb];
+	ref_count = &map_1gb->ref_count[idx_1gb];
+
+	if (*ref_count == VTOPHYS_MAX_REF_COUNT) {
+#ifdef DEBUG
+		fprintf(stderr, "ref count for %p already at %d\n",
+			(void *)vaddr, VTOPHYS_MAX_REF_COUNT);
+#endif
+		return;
+	}
+
+	map_2mb->translation_2mb = translation;
+
+	(*ref_count)++;
+}
+
+static void
+spdk_mem_map_unregister(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size)
+{
+	uint64_t vfn_2mb;
+	struct map_1gb *map_1gb;
+	uint64_t idx_1gb;
+	struct map_2mb *map_2mb;
+	uint16_t *ref_count;
+
+	/* For now, only 2 MB registrations are supported */
+	assert(size == 2 * 1024 * 1024);
+	assert((vaddr & MASK_2MB) == 0);
+
+	vfn_2mb = vaddr >> SHIFT_2MB;
+
+	map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
+	if (!map_1gb) {
+#ifdef DEBUG
+		fprintf(stderr, "could not get %p map\n", (void *)vaddr);
+#endif
+		return;
+	}
+
+	idx_1gb = MAP_1GB_IDX(vfn_2mb);
+	map_2mb = &map_1gb->map[idx_1gb];
+	ref_count = &map_1gb->ref_count[idx_1gb];
+
+	if (*ref_count == 0) {
+#ifdef DEBUG
+		fprintf(stderr, "vaddr %p not registered\n", (void *)vaddr);
+#endif
+		return;
+	}
+
+	(*ref_count)--;
+	if (*ref_count == 0) {
+		map_2mb->translation_2mb = SPDK_VTOPHYS_ERROR;
+	}
+}
+
+static uint64_t
+spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr)
+{
+	const struct map_1gb *map_1gb;
+	const struct map_2mb *map_2mb;
+	uint64_t idx_128tb;
+	uint64_t idx_1gb;
+	uint64_t vfn_2mb;
+
+	if (spdk_unlikely(vaddr & ~MASK_128TB)) {
+#ifdef DEBUG
+		printf("invalid usermode virtual address %p\n", (void *)vaddr);
+#endif
+		return SPDK_VTOPHYS_ERROR;
+	}
+
+	vfn_2mb = vaddr >> SHIFT_2MB;
+	idx_128tb = MAP_128TB_IDX(vfn_2mb);
+	idx_1gb = MAP_1GB_IDX(vfn_2mb);
+
+	map_1gb = map->map_128tb.map[idx_128tb];
+	if (spdk_unlikely(!map_1gb)) {
+		return SPDK_VTOPHYS_ERROR;
+	}
+
+	map_2mb = &map_1gb->map[idx_1gb];
+
+	return map_2mb->translation_2mb;
+}
+
 static uint64_t
 vtophys_get_dpdk_paddr(void *vaddr)
 {
@@ -187,11 +302,6 @@ vtophys_get_paddr(uint64_t vaddr)
 static void
 _spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr)
 {
-	struct map_1gb *map_1gb;
-	uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb);
-	struct map_2mb *map_2mb;
-	uint16_t *ref_count;
-
 	if (paddr & MASK_2MB) {
 #ifdef DEBUG
 		fprintf(stderr, "invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr);
@@ -199,60 +309,13 @@ _spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr)
 		return;
 	}
 
-	map_1gb = vtophys_get_map_1gb(vfn_2mb);
+	spdk_mem_map_register(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024, paddr);
-	if (!map_1gb) {
-#ifdef DEBUG
-		fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb);
-#endif
-		return;
-	}
-
-	map_2mb = &map_1gb->map[idx_1gb];
-	ref_count = &map_1gb->ref_count[idx_1gb];
-
-	if (*ref_count == VTOPHYS_MAX_REF_COUNT) {
-#ifdef DEBUG
-		fprintf(stderr, "ref count for %p already at %d\n",
-			(void *)(vfn_2mb << SHIFT_2MB), VTOPHYS_MAX_REF_COUNT);
-#endif
-		return;
-	}
-
-	map_2mb->paddr_2mb = paddr;
-
-	(*ref_count)++;
 }
 
 static void
 _spdk_vtophys_unregister_one(uint64_t vfn_2mb)
 {
-	struct map_1gb *map_1gb;
+	spdk_mem_map_unregister(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024);
-	uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb);
-	struct map_2mb *map_2mb;
-	uint16_t *ref_count;
-
-	map_1gb = vtophys_get_map_1gb(vfn_2mb);
-	if (!map_1gb) {
-#ifdef DEBUG
-		fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb);
-#endif
-		return;
-	}
-
-	map_2mb = &map_1gb->map[idx_1gb];
-	ref_count = &map_1gb->ref_count[idx_1gb];
-
-	if (map_2mb->paddr_2mb == SPDK_VTOPHYS_ERROR || *ref_count == 0) {
-#ifdef DEBUG
-		fprintf(stderr, "vaddr %p not registered\n", (void *)(vfn_2mb << SHIFT_2MB));
-#endif
-		return;
-	}
-
-	(*ref_count)--;
-	if (*ref_count == 0) {
-		map_2mb->paddr_2mb = SPDK_VTOPHYS_ERROR;
-	}
 }
 
 void
@@ -347,32 +410,11 @@ spdk_vtophys_register_dpdk_mem(void)
 uint64_t
 spdk_vtophys(void *buf)
 {
-	struct map_1gb *map_1gb;
+	uint64_t vaddr, paddr_2mb;
-	struct map_2mb *map_2mb;
-	uint64_t idx_128tb;
-	uint64_t idx_1gb;
-	uint64_t vaddr, vfn_2mb, paddr_2mb;
 
 	vaddr = (uint64_t)buf;
-	if (spdk_unlikely(vaddr & ~MASK_128TB)) {
-#ifdef DEBUG
-		printf("invalid usermode virtual address %p\n", buf);
-#endif
-		return SPDK_VTOPHYS_ERROR;
-	}
 
-	vfn_2mb = vaddr >> SHIFT_2MB;
+	paddr_2mb = spdk_mem_map_translate(&g_vtophys_map, vaddr);
-	idx_128tb = MAP_128TB_IDX(vfn_2mb);
-	idx_1gb = MAP_1GB_IDX(vfn_2mb);
-
-	map_1gb = vtophys_map_128tb.map[idx_128tb];
-	if (spdk_unlikely(!map_1gb)) {
-		return SPDK_VTOPHYS_ERROR;
-	}
-
-	map_2mb = &map_1gb->map[idx_1gb];
-
-	paddr_2mb = map_2mb->paddr_2mb;
 
 	/*
 	 * SPDK_VTOPHYS_ERROR has all bits set, so if the lookup returned SPDK_VTOPHYS_ERROR,