[compiler-rt] [sanitizer_common] Implement address sanitizer on AIX: stack unwinding (PR #138188)

[Jake Egan via llvm-commits]

https://github.com/jakeegan updated https://github.com/llvm/llvm-project/pull/138188

>From 6f86bdd1bbf1d597a55f3f3caae13910961fa051 Mon Sep 17 00:00:00 2001
From: Jake Egan <jake.egan at ibm.com>
Date: Thu, 1 May 2025 15:37:10 -0400
Subject: [PATCH 1/3] unwinding

---
 .../lib/sanitizer_common/CMakeLists.txt       |  1 +
 .../sanitizer_common/sanitizer_unwind_aix.cpp | 66 +++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp

diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
index 6e6dfd2f33ebf..96c23c6d8ab82 100644
--- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt
+++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
@@ -97,6 +97,7 @@ set(SANITIZER_SYMBOLIZER_SOURCES
   sanitizer_symbolizer_report_fuchsia.cpp
   sanitizer_symbolizer_win.cpp
   sanitizer_thread_history.cpp
+  sanitizer_unwind_aix.cpp
   sanitizer_unwind_linux_libcdep.cpp
   sanitizer_unwind_fuchsia.cpp
   sanitizer_unwind_win.cpp
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
new file mode 100644
index 0000000000000..45b4b1f3d5c9d
--- /dev/null
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
@@ -0,0 +1,66 @@
+//===-- sanitizer_unwind_aix.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the unwind.h-based (aka "slow") stack unwinding routines
+// available to the tools on AIX.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_AIX
+#  include <unwind.h>
+
+#  include "sanitizer_common.h"
+#  include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+struct UnwindTraceArg {
+  BufferedStackTrace *stack;
+  u32 max_depth;
+};
+
+_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
+  UnwindTraceArg *arg = (UnwindTraceArg *)param;
+  CHECK_LT(arg->stack->size, arg->max_depth);
+  uptr pc = _Unwind_GetIP(ctx);
+  // On AIX 32-bit and 64-bit, address smaller than 0x0fffffff is for kernel.
+  if (pc <= 0x0fffffff)
+    return _URC_NORMAL_STOP;
+  arg->stack->trace_buffer[arg->stack->size++] = pc;
+  if (arg->stack->size == arg->max_depth)
+    return _URC_NORMAL_STOP;
+  return _URC_NO_REASON;
+}
+
+void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
+  CHECK_GE(max_depth, 2);
+  size = 0;
+  UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
+  _Unwind_Backtrace(Unwind_Trace, &arg);
+  // We need to pop a few frames so that pc is on top.
+  uptr to_pop = LocatePcInTrace(pc);
+  // trace_buffer[0] belongs to the current function so we always pop it,
+  // unless there is only 1 frame in the stack trace (1 frame is always better
+  // than 0!).
+  // 1-frame stacks don't normally happen, but this depends on the actual
+  // unwinder implementation (libgcc, libunwind, etc) which is outside of our
+  // control.
+  if (to_pop == 0 && size > 1)
+    to_pop = 1;
+
+  PopStackFrames(to_pop);
+  trace_buffer[0] = pc;
+}
+
+void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
+  UnwindSlow(pc, max_depth);
+}
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_AIX

>From 9924f3bb0917af497fd2a64d8ad8e418787502d8 Mon Sep 17 00:00:00 2001
From: Jake Egan <jake.egan at ibm.com>
Date: Mon, 7 Jul 2025 22:07:06 -0400
Subject: [PATCH 2/3] Address comments

---
 compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
index 45b4b1f3d5c9d..02f287dc7b6e9 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
@@ -25,11 +25,11 @@ struct UnwindTraceArg {
   u32 max_depth;
 };
 
-_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
+static _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
   UnwindTraceArg *arg = (UnwindTraceArg *)param;
   CHECK_LT(arg->stack->size, arg->max_depth);
   uptr pc = _Unwind_GetIP(ctx);
-  // On AIX 32-bit and 64-bit, address smaller than 0x0fffffff is for kernel.
+  // On AIX 32-bit and 64-bit, addresses up through 0x0fffffff are for kernel.
   if (pc <= 0x0fffffff)
     return _URC_NORMAL_STOP;
   arg->stack->trace_buffer[arg->stack->size++] = pc;
@@ -59,6 +59,7 @@ void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
 }
 
 void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
+  CHECK(context);
   UnwindSlow(pc, max_depth);
 }
 }  // namespace __sanitizer

>From 3f183319b886250e9f2d161500198579d6a5d70b Mon Sep 17 00:00:00 2001
From: Jake Egan <jake.egan at ibm.com>
Date: Tue, 8 Jul 2025 09:55:44 -0400
Subject: [PATCH 3/3] Fix formatting

---
 compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
index 02f287dc7b6e9..0aaf6bdabadfd 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_aix.cpp
@@ -25,7 +25,8 @@ struct UnwindTraceArg {
   u32 max_depth;
 };
 
-static _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
+static _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx,
+                                        void *param) {
   UnwindTraceArg *arg = (UnwindTraceArg *)param;
   CHECK_LT(arg->stack->size, arg->max_depth);
   uptr pc = _Unwind_GetIP(ctx);