diff --git a/include/spdk/barrier.h b/include/spdk/barrier.h index 20a4cacd5..acae360c7 100644 --- a/include/spdk/barrier.h +++ b/include/spdk/barrier.h @@ -48,78 +48,65 @@ extern "C" { /** Compiler memory barrier */ #define spdk_compiler_barrier() __asm volatile("" ::: "memory") -/** Write memory barrier */ -#ifdef __PPC64__ -#define spdk_wmb() __asm volatile("sync" ::: "memory") -#elif defined(__aarch64__) -#define spdk_wmb() __asm volatile("dsb st" ::: "memory") -#elif defined(__i386__) || defined(__x86_64__) -#define spdk_wmb() __asm volatile("sfence" ::: "memory") -#else -#define spdk_wmb() -#error Unknown architecture -#endif - /** Read memory barrier */ -#ifdef __PPC64__ -#define spdk_rmb() __asm volatile("sync" ::: "memory") -#elif defined(__aarch64__) -#define spdk_rmb() __asm volatile("dsb ld" ::: "memory") -#elif defined(__i386__) || defined(__x86_64__) -#define spdk_rmb() __asm volatile("lfence" ::: "memory") -#else -#define spdk_rmb() -#error Unknown architecture -#endif +#define spdk_rmb() _spdk_rmb() + +/** Write memory barrier */ +#define spdk_wmb() _spdk_wmb() /** Full read/write memory barrier */ -#ifdef __PPC64__ -#define spdk_mb() __asm volatile("sync" ::: "memory") -#elif defined(__aarch64__) -#define spdk_mb() __asm volatile("dsb sy" ::: "memory") -#elif defined(__i386__) || defined(__x86_64__) -#define spdk_mb() __asm volatile("mfence" ::: "memory") -#else -#define spdk_mb() -#error Unknown architecture -#endif +#define spdk_mb() _spdk_mb() /** SMP read memory barrier. */ -#ifdef __PPC64__ -#define spdk_smp_rmb() __asm volatile("lwsync" ::: "memory") -#elif defined(__aarch64__) -#define spdk_smp_rmb() __asm volatile("dmb ishld" ::: "memory") -#elif defined(__i386__) || defined(__x86_64__) -#define spdk_smp_rmb() spdk_compiler_barrier() -#else -#define spdk_smp_rmb() -#error Unknown architecture -#endif +#define spdk_smp_rmb() _spdk_smp_rmb() /** SMP write memory barrier. */ -#ifdef __PPC64__ -#define spdk_smp_wmb() __asm volatile("lwsync" ::: "memory") -#elif defined(__aarch64__) -#define spdk_smp_wmb() __asm volatile("dmb ishst" ::: "memory") -#elif defined(__i386__) || defined(__x86_64__) -#define spdk_smp_wmb() spdk_compiler_barrier() -#else -#define spdk_smp_wmb() -#error Unknown architecture -#endif +#define spdk_smp_wmb() _spdk_smp_wmb() /** SMP read/write memory barrier. */ +#define spdk_smp_mb() _spdk_smp_mb() + #ifdef __PPC64__ -#define spdk_smp_mb() spdk_mb() + +#define _spdk_rmb() __asm volatile("sync" ::: "memory") +#define _spdk_wmb() __asm volatile("sync" ::: "memory") +#define _spdk_mb() __asm volatile("sync" ::: "memory") +#define _spdk_smp_rmb() __asm volatile("lwsync" ::: "memory") +#define _spdk_smp_wmb() __asm volatile("lwsync" ::: "memory") +#define _spdk_smp_mb() spdk_mb() + #elif defined(__aarch64__) -#define spdk_smp_mb() __asm volatile("dmb ish" ::: "memory") -#elif defined(__x86_64__) -#define spdk_smp_mb() __asm volatile("lock addl $0, -128(%%rsp); " ::: "memory"); + +#define _spdk_rmb() __asm volatile("dsb ld" ::: "memory") +#define _spdk_wmb() __asm volatile("dsb st" ::: "memory") +#define _spdk_mb() __asm volatile("dsb sy" ::: "memory") +#define _spdk_smp_rmb() __asm volatile("dmb ishld" ::: "memory") +#define _spdk_smp_wmb() __asm volatile("dmb ishst" ::: "memory") +#define _spdk_smp_mb() __asm volatile("dmb ish" ::: "memory") + +#elif defined(__i386__) || defined(__x86_64__) + +#define _spdk_rmb() __asm volatile("lfence" ::: "memory") +#define _spdk_wmb() __asm volatile("sfence" ::: "memory") +#define _spdk_mb() __asm volatile("mfence" ::: "memory") +#define _spdk_smp_rmb() spdk_compiler_barrier() +#define _spdk_smp_wmb() spdk_compiler_barrier() +#if defined(__x86_64__) +#define _spdk_smp_mb() __asm volatile("lock addl $0, -128(%%rsp); " ::: "memory"); #elif defined(__i386__) -#define spdk_smp_mb() __asm volatile("lock addl $0, -128(%%esp); " ::: "memory"); +#define _spdk_smp_mb() __asm volatile("lock addl $0, -128(%%esp); " ::: "memory"); +#endif + #else -#define spdk_smp_mb() + +#define _spdk_rmb() +#define _spdk_wmb() +#define _spdk_mb() +#define _spdk_smp_rmb() +#define _spdk_smp_wmb() +#define _spdk_smp_mb() #error Unknown architecture + #endif #ifdef __cplusplus