[llvm] [StackColoring] Change TotalStackSize from unsigned to int64_t (PR #208671)

[Philipp Rados via llvm-commits]

https://github.com/prados-oc created https://github.com/llvm/llvm-project/pull/208671

StackColoring tracks the total size of the stack as `unsigned int`. This will wrap around, even on 64-bit systems, if the stack is greater than that resulting in a wrong size. This can happen on both PPC and RISCV64.

>From 57672ce832c4d2215bb661071c6d285c28260629 Mon Sep 17 00:00:00 2001
From: Philipp Rados <philipp.rados at openchip.com>
Date: Thu, 9 Jul 2026 20:53:43 +0200
Subject: [PATCH 1/2] Add previously fault test

---
 .../CodeGen/RISCV/stack-coloring-large-stack.ll | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/stack-coloring-large-stack.ll

diff --git a/llvm/test/CodeGen/RISCV/stack-coloring-large-stack.ll b/llvm/test/CodeGen/RISCV/stack-coloring-large-stack.ll
new file mode 100644
index 0000000000000..2d00e7e3bc630
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stack-coloring-large-stack.ll
@@ -0,0 +1,17 @@
+; REQUIRES: asserts
+; RUN: llc < %s -O3 -mtriple=riscv64 -debug-only=stack-coloring 2>&1 | FileCheck %s
+
+declare void @foo(...)
+
+; CHECK: Slot #0 - 33179869176 bytes.
+; CHECK: Slot #1 - 147483648 bytes.
+; CHECK: Total Stack size: 33327352824 bytes
+define dso_local void @stack_bigger_than_32bit_unsigned() {
+entry:
+  %a = alloca [4147483647 x i64], align 8
+  %b = alloca [147483648 x i8], align 1
+  %arraydecay = getelementptr inbounds [4147483647 x i64], ptr %a, i64 0, i64 0
+  %arraydecay1 = getelementptr inbounds [147483648 x i8], ptr %b, i64 0, i64 0
+  call void @foo(ptr noundef %arraydecay, ptr noundef %arraydecay1)
+  ret void
+}

>From 4a72dd0f27320653ac197f51f9c98479f037a4c0 Mon Sep 17 00:00:00 2001
From: Philipp Rados <philipp.rados at openchip.com>
Date: Thu, 9 Jul 2026 20:54:13 +0200
Subject: [PATCH 2/2] [StackColoring] Change TotalStackSize from unsigned to
 int64_t

---
 llvm/lib/CodeGen/StackColoring.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp
index ea4c49dae8260..ea1f33f791765 100644
--- a/llvm/lib/CodeGen/StackColoring.cpp
+++ b/llvm/lib/CodeGen/StackColoring.cpp
@@ -1224,7 +1224,7 @@ bool StackColoring::run(MachineFunction &Func, bool OnlyRemoveMarkers) {
 
   unsigned NumMarkers = collectMarkers(NumSlots);
 
-  unsigned TotalSize = 0;
+  int64_t TotalSize = 0;
   LLVM_DEBUG(dbgs() << "Found " << NumMarkers << " markers and " << NumSlots
                     << " slots\n");
   LLVM_DEBUG(dbgs() << "Slot structure:\n");
@@ -1270,7 +1270,7 @@ bool StackColoring::run(MachineFunction &Func, bool OnlyRemoveMarkers) {
   // Maps old slots to new slots.
   DenseMap<int, int> SlotRemap;
   unsigned RemovedSlots = 0;
-  unsigned ReducedSize = 0;
+  int64_t ReducedSize = 0;
 
   // Do not bother looking at empty intervals.
   for (unsigned I = 0; I < NumSlots; ++I) {