From 5d926c48ea6b11df51a52e1ae78b4e8965f48ad9 Mon Sep 17 00:00:00 2001 From: Rui Chang Date: Thu, 11 Nov 2021 18:15:48 +0800 Subject: [PATCH] nvmf/vfio-user: Fix doorbell polling not working on some ARM platform On aarch64 platforms, doorbells update from guest VM may not be seen on SPDK target side. This is because there is memory type mismatch situation here. That is on guest VM side, the doorbells are treated as device memory while on SPDK target side, it is treated as normal memory. And this situation cause problem on ARM platform. Refer to "https://developer.arm.com/documentation/102376/0100/ Memory-aliasing-and-mismatched-memory-types". Only using spdk_mb() cannot fix this. Use "dc civac" to invalidate cache may solve this. Profiling data did not show big performance degradataion. Signed-off-by: Rui Chang Change-Id: I9a18718f8c4307b3007b18c32ab02e6796548958 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10222 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Reviewed-by: Ben Walker Reviewed-by: Changpeng Liu --- include/spdk/barrier.h | 7 +++++++ lib/nvmf/vfio_user.c | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/include/spdk/barrier.h b/include/spdk/barrier.h index acae360c7..eb1992ba2 100644 --- a/include/spdk/barrier.h +++ b/include/spdk/barrier.h @@ -66,6 +66,9 @@ extern "C" { /** SMP read/write memory barrier. */ #define spdk_smp_mb() _spdk_smp_mb() +/** Invalidate data cache, input is data pointer */ +#define spdk_ivdt_dcache(pdata) _spdk_ivdt_dcache(pdata) + #ifdef __PPC64__ #define _spdk_rmb() __asm volatile("sync" ::: "memory") @@ -74,6 +77,7 @@ extern "C" { #define _spdk_smp_rmb() __asm volatile("lwsync" ::: "memory") #define _spdk_smp_wmb() __asm volatile("lwsync" ::: "memory") #define _spdk_smp_mb() spdk_mb() +#define _spdk_ivdt_dcache(pdata) #elif defined(__aarch64__) @@ -83,6 +87,7 @@ extern "C" { #define _spdk_smp_rmb() __asm volatile("dmb ishld" ::: "memory") #define _spdk_smp_wmb() __asm volatile("dmb ishst" ::: "memory") #define _spdk_smp_mb() __asm volatile("dmb ish" ::: "memory") +#define _spdk_ivdt_dcache(pdata) asm volatile("dc civac, %0" : : "r"(pdata) : "memory"); #elif defined(__i386__) || defined(__x86_64__) @@ -96,6 +101,7 @@ extern "C" { #elif defined(__i386__) #define _spdk_smp_mb() __asm volatile("lock addl $0, -128(%%esp); " ::: "memory"); #endif +#define _spdk_ivdt_dcache(pdata) #else @@ -105,6 +111,7 @@ extern "C" { #define _spdk_smp_rmb() #define _spdk_smp_wmb() #define _spdk_smp_mb() +#define _spdk_ivdt_dcache(pdata) #error Unknown architecture #endif diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index 43262d94d..ed0525c72 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -2836,6 +2836,18 @@ nvmf_vfio_user_qpair_poll(struct nvmf_vfio_user_qpair *qpair) ctrlr = qpair->ctrlr; + /* On aarch64 platforms, doorbells update from guest VM may not be seen + * on SPDK target side. This is because there is memory type mismatch + * situation here. That is on guest VM side, the doorbells are treated as + * device memory while on SPDK target side, it is treated as normal + * memory. And this situation cause problem on ARM platform. + * Refer to "https://developer.arm.com/documentation/102376/0100/ + * Memory-aliasing-and-mismatched-memory-types". Only using spdk_mb() + * cannot fix this. Use "dc civac" to invalidate cache may solve + * this. + */ + spdk_ivdt_dcache(tdbl(ctrlr, &qpair->sq)); + /* Load-Acquire. */ new_tail = *tdbl(ctrlr, &qpair->sq);