env: define minimum alignment for dma-able memory

DPDK defines the minimum alignment as "suitable for any
kind of variable (in the same manner as malloc())", but
internally the alignment is always rounded up to the
cache line size, even if the requested alignment is 0.

We would like to start relying on this behavior in FTL,
where lba maps are allocated using DMA-able memory and
are constantly looked up or modified by different threads.
By having the lba maps unaligned, we risk having those
threads pollute each other's cache lines.

Rather than enforcing this memory alignment in FTL, we
do it in spdk_*malloc directly. In general it makes
sense to have DMA-able memory always cache-line-size
aligned for the same reason as above.

Change-Id: Ib6edda4a7bf3f4952eb1875a4e1753be96bed642
Signed-off-by: Mateusz Kozlowski <mateusz.kozlowski@intel.com>
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/460329
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

include/spdk/env.h

@@ -94,10 +94,9 @@ struct spdk_env_opts {
  * with the given size, alignment and socket id.
  *
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  * \param phys_addr **Deprecated**. Please use spdk_vtophys() for retrieving physical
  * addresses. A pointer to the variable to hold the physical address of
  * the allocated buffer is passed. If NULL, the physical address is not returned.
@@ -115,10 +114,9 @@ void *spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id,
  * with the given size, alignment and socket id. Also, the buffer will be zeroed.
  *
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  * \param phys_addr **Deprecated**. Please use spdk_vtophys() for retrieving physical
  * addresses. A pointer to the variable to hold the physical address of
  * the allocated buffer is passed. If NULL, the physical address is not returned.
@@ -136,10 +134,9 @@ void *spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id
  *
  * \param buf Buffer to resize.
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  *
  * \return a pointer to the resized memory buffer.
  */
@@ -180,10 +177,9 @@ void spdk_env_fini(void);
  * Allocate a pinned memory buffer with the given size and alignment.
  *
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  * \param phys_addr A pointer to the variable to hold the physical address of
  * the allocated buffer is passed. If NULL, the physical address is not returned.
  *
@@ -195,10 +191,9 @@ void *spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr);
  * Allocate a pinned, memory buffer with the given size, alignment and socket id.
  *
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  * \param phys_addr A pointer to the variable to hold the physical address of
  * the allocated buffer is passed. If NULL, the physical address is not returned.
  * \param socket_id Socket ID to allocate memory on, or SPDK_ENV_SOCKET_ID_ANY
@@ -213,10 +208,9 @@ void *spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int
  * will be zeroed.
  *
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  * \param phys_addr A pointer to the variable to hold the physical address of
  * the allocated buffer is passed. If NULL, the physical address is not returned.
  *
@@ -229,10 +223,9 @@ void *spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr);
  * The buffer will be zeroed.
  *
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  * \param phys_addr A pointer to the variable to hold the physical address of
  * the allocated buffer is passed. If NULL, the physical address is not returned.
  * \param socket_id Socket ID to allocate memory on, or SPDK_ENV_SOCKET_ID_ANY
@@ -248,10 +241,9 @@ void *spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, in
  *
  * \param buf Buffer to resize.
  * \param size Size in bytes.
- * \param align Alignment value for the allocated memory. If '0', the allocated
+ * \param align If non-zero, the allocated buffer is aligned to a multiple of
- * buffer is suitably aligned (in the same manner as malloc()). Otherwise, the
+ * align. In this case, it must be a power of two. The returned buffer is always
- * allocated buffer is aligned to the multiple of align. In this case, it must
+ * aligned to at least cache line size.
- * be a power of two.
  * \param phys_addr A pointer to the variable to hold the physical address of
  * the allocated buffer is passed. If NULL, the physical address is not returned.
  *

lib/env_dpdk/env.c

@@ -32,6 +32,7 @@
  */
 
 #include "spdk/stdinc.h"
+#include "spdk/util.h"
 
 #include "env_internal.h"
 
@@ -58,11 +59,14 @@ virt_to_phys(void *vaddr)
 void *
 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags)
 {
+	void *buf;
+
 	if (flags == 0) {
 		return NULL;
 	}
 
-	void *buf = rte_malloc_socket(NULL, size, align, socket_id);
+	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
+	buf = rte_malloc_socket(NULL, size, align, socket_id);
 	if (buf && phys_addr) {
 #ifdef DEBUG
 		fprintf(stderr, "phys_addr param in spdk_*malloc() is deprecated\n");
@@ -85,6 +89,7 @@ spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint
 void *
 spdk_realloc(void *buf, size_t size, size_t align)
 {
+	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
 	return rte_realloc(buf, size, align);
 }
 
@@ -121,7 +126,10 @@ spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr)
 void *
 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr)
 {
-	void *new_buf = rte_realloc(buf, size, align);
+	void *new_buf;
+
+	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
+	new_buf = rte_realloc(buf, size, align);
 	if (new_buf && phys_addr) {
 		*phys_addr = virt_to_phys(new_buf);
 	}