From 531258aa51000ab18cf4b3dd211673e5adb201e5 Mon Sep 17 00:00:00 2001
From: Mike Gerdts <mgerdts@nvidia.com>
Date: Fri, 16 Dec 2022 12:48:42 -0600
Subject: [PATCH] thread: get debug stack traces on spinlocks

To help debug spinlocks, capture stack traces as spinlocks are used.
Future commits in this series will make debugging with these stack
traces easier.

Signed-off-by: Mike Gerdts <mgerdts@nvidia.com>
Change-Id: I597b730ca771ea3c5b831f5ba4058d359215f7f6
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15998
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
---
 include/spdk/thread.h |  1 +
 lib/thread/thread.c   | 55 +++++++++++++++++++++++++++++++++++++++++++
 mk/spdk.common.mk     |  2 +-
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/include/spdk/thread.h b/include/spdk/thread.h
index f036b19d8..198946a89 100644
--- a/include/spdk/thread.h
+++ b/include/spdk/thread.h
@@ -904,6 +904,7 @@ bool spdk_interrupt_mode_is_enabled(void);
 struct spdk_spinlock {
 	pthread_spinlock_t spinlock;
 	struct spdk_thread *thread;
+	struct spdk_spinlock_internal *internal;
 };
 
 /**
diff --git a/lib/thread/thread.c b/lib/thread/thread.c
index 4ea28055e..bc376503c 100644
--- a/lib/thread/thread.c
+++ b/lib/thread/thread.c
@@ -25,6 +25,11 @@
 #ifdef __linux__
 #include <sys/timerfd.h>
 #include <sys/eventfd.h>
+#include <execinfo.h>
+#endif
+
+#ifdef __FreeBSD__
+#include <execinfo.h>
 #endif
 
 #define SPDK_MSG_BATCH_SIZE		8
@@ -2876,6 +2881,48 @@ spdk_interrupt_mode_is_enabled(void)
 	return g_interrupt_mode;
 }
 
+#define SSPIN_DEBUG_STACK_FRAMES 16
+
+struct sspin_stack {
+	void *addrs[SSPIN_DEBUG_STACK_FRAMES];
+	uint32_t depth;
+};
+
+struct spdk_spinlock_internal {
+	struct sspin_stack init_stack;
+	struct sspin_stack lock_stack;
+	struct sspin_stack unlock_stack;
+};
+
+static void
+sspin_init_internal(struct spdk_spinlock *sspin)
+{
+#ifdef DEBUG
+	sspin->internal = calloc(1, sizeof(*sspin->internal));
+#endif
+}
+
+static void
+sspin_fini_internal(struct spdk_spinlock *sspin)
+{
+#ifdef DEBUG
+	free(sspin->internal);
+	sspin->internal = NULL;
+#endif
+}
+
+#ifdef DEBUG
+#define SSPIN_GET_STACK(sspin, which) \
+	do { \
+		if (sspin->internal != NULL) { \
+			struct sspin_stack *stack = &sspin->internal->which ## _stack; \
+			stack->depth = backtrace(stack->addrs, SPDK_COUNTOF(stack->addrs)); \
+		} \
+	} while (0)
+#else
+#define SSPIN_GET_STACK(sspin, which) do { } while (0)
+#endif
+
 void
 spdk_spin_init(struct spdk_spinlock *sspin)
 {
@@ -2884,6 +2931,8 @@ spdk_spin_init(struct spdk_spinlock *sspin)
 	memset(sspin, 0, sizeof(*sspin));
 	rc = pthread_spin_init(&sspin->spinlock, PTHREAD_PROCESS_PRIVATE);
 	SPIN_ASSERT_RETURN_VOID(rc == 0, SPIN_ERR_PTHREAD);
+	sspin_init_internal(sspin);
+	SSPIN_GET_STACK(sspin, init);
 }
 
 void
@@ -2895,6 +2944,8 @@ spdk_spin_destroy(struct spdk_spinlock *sspin)
 
 	rc = pthread_spin_destroy(&sspin->spinlock);
 	SPIN_ASSERT_RETURN_VOID(rc == 0, SPIN_ERR_PTHREAD);
+
+	sspin_fini_internal(sspin);
 }
 
 void
@@ -2911,6 +2962,8 @@ spdk_spin_lock(struct spdk_spinlock *sspin)
 
 	sspin->thread = thread;
 	sspin->thread->lock_count++;
+
+	SSPIN_GET_STACK(sspin, lock);
 }
 
 void
@@ -2926,6 +2979,8 @@ spdk_spin_unlock(struct spdk_spinlock *sspin)
 	thread->lock_count--;
 	sspin->thread = NULL;
 
+	SSPIN_GET_STACK(sspin, unlock);
+
 	rc = pthread_spin_unlock(&sspin->spinlock);
 	SPIN_ASSERT_RETURN_VOID(rc == 0, SPIN_ERR_PTHREAD);
 }
diff --git a/mk/spdk.common.mk b/mk/spdk.common.mk
index 5c7d26fb6..b8013a5c0 100644
--- a/mk/spdk.common.mk
+++ b/mk/spdk.common.mk
@@ -154,7 +154,7 @@ endif
 SYS_LIBS =
 
 ifeq ($(OS),FreeBSD)
-SYS_LIBS += -L/usr/local/lib
+SYS_LIBS += -lexecinfo -L/usr/local/lib
 COMMON_CFLAGS += -I/usr/local/include
 endif