[llvm] [Hexagon] Add probe-stack=inline-asm support for stack clash protection (PR #190568)
Brian Cain via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 24 07:25:54 PDT 2026
https://github.com/androm3da updated https://github.com/llvm/llvm-project/pull/190568
>From 6fde6d6826b08e9963bd34741f9535ce0aece0e9 Mon Sep 17 00:00:00 2001
From: Brian Cain <brian.cain at oss.qualcomm.com>
Date: Sun, 5 Apr 2026 15:55:55 -0700
Subject: [PATCH 1/3] [Hexagon] Add probe-stack=inline-asm support for stack
clash protection
Large stack allocations can skip over guard pages, causing stack clash
vulnerabilities. The probe-stack=inline-asm function attribute tells
LLVM to emit inline probing code that touches each page during stack
allocation, ensuring guard pages are hit.
`framelimit` provides hardware bounds checking, but only on the allocframe
instruction itself. SP decrements via A2_addi -- used for frames >= 16k bytes
and all no-FP prologues bypass it. Software probing closes that gap.
When the attribute is present and the frame size exceeds the probe size,
the prologue now emits a PS_probed_stackalloc pseudo that inlineStackProbe()
expands into a compare-and-branch loop:
r29 = add(r29, #-ProbeSize)
memw(r29+#0) = #0
p0 = cmp.gtu(r29, r28)
if (p0) jump LoopMBB
r29 = r28
Both the frame-pointer and no-frame-pointer prologue paths are handled.
The stack-probe-size attribute is respected for custom probe sizes.
---
.../Target/Hexagon/HexagonFrameLowering.cpp | 148 +++++++++++++++-
.../lib/Target/Hexagon/HexagonFrameLowering.h | 3 +
.../Target/Hexagon/HexagonISelLowering.cpp | 18 ++
llvm/lib/Target/Hexagon/HexagonISelLowering.h | 3 +
llvm/lib/Target/Hexagon/HexagonPseudo.td | 6 +
llvm/test/CodeGen/Hexagon/stack-probing.ll | 166 ++++++++++++++++++
6 files changed, 340 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/Hexagon/stack-probing.ll
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
index 0f73efc243986..2bd40d0eb6f28 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -9,6 +9,7 @@
#include "HexagonFrameLowering.h"
#include "HexagonBlockRanges.h"
+#include "HexagonISelLowering.h"
#include "HexagonInstrInfo.h"
#include "HexagonMachineFunctionInfo.h"
#include "HexagonRegisterInfo.h"
@@ -746,9 +747,29 @@ void HexagonFrameLowering::insertPrologueInBlock(MachineBasicBlock &MBB,
.addExternalSymbol("__runtime_stack_check");
} else if (NumBytes > 0) {
assert(alignTo(NumBytes, 8) == NumBytes);
- BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::A2_addi), SP)
- .addReg(SP)
- .addImm(-int(NumBytes));
+ auto *TLI = HST.getTargetLowering();
+ bool NeedsProbing = TLI->hasInlineStackProbe(MF);
+ unsigned ProbeSize = 0;
+ if (NeedsProbing) {
+ Align StackAlign = getStackAlign();
+ ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
+ }
+ if (NeedsProbing && NumBytes > ProbeSize) {
+ // Compute target SP in R28 (caller-saved scratch).
+ BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::A2_addi), Hexagon::R28)
+ .addReg(SP)
+ .addImm(-int(NumBytes))
+ .setMIFlag(MachineInstr::FrameSetup);
+ // Emit pseudo to be expanded by inlineStackProbe().
+ BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::PS_probed_stackalloc))
+ .addReg(Hexagon::R28)
+ .setMIFlag(MachineInstr::FrameSetup);
+ } else {
+ BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::A2_addi), SP)
+ .addReg(SP)
+ .addImm(-int(NumBytes))
+ .setMIFlag(MachineInstr::FrameSetup);
+ }
}
}
@@ -892,7 +913,34 @@ void HexagonFrameLowering::insertAllocframe(MachineBasicBlock &MBB,
DebugLoc dl = MBB.findDebugLoc(InsertPt);
Register SP = HRI.getStackRegister();
- if (NumBytes >= ALLOCFRAME_MAX) {
+ auto *TLI = HST.getTargetLowering();
+ bool NeedsProbing = TLI->hasInlineStackProbe(MF) && NumBytes > 0;
+ unsigned ProbeSize = 0;
+ if (NeedsProbing) {
+ Align StackAlign = getStackAlign();
+ ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
+ }
+
+ if (NeedsProbing && NumBytes > ProbeSize) {
+ // Emit allocframe(#0) to save FP/LR only.
+ BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::S2_allocframe))
+ .addDef(SP)
+ .addReg(SP)
+ .addImm(0)
+ .addMemOperand(MMO)
+ .setMIFlag(MachineInstr::FrameSetup);
+
+ // Compute target SP in R28 (caller-saved scratch).
+ BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::A2_addi), Hexagon::R28)
+ .addReg(SP)
+ .addImm(-int(NumBytes))
+ .setMIFlag(MachineInstr::FrameSetup);
+
+ // Emit pseudo to be expanded by inlineStackProbe().
+ BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::PS_probed_stackalloc))
+ .addReg(Hexagon::R28)
+ .setMIFlag(MachineInstr::FrameSetup);
+ } else if (NumBytes >= ALLOCFRAME_MAX) {
// Emit allocframe(#0).
BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::S2_allocframe))
.addDef(SP)
@@ -917,6 +965,98 @@ void HexagonFrameLowering::insertAllocframe(MachineBasicBlock &MBB,
}
}
+void HexagonFrameLowering::inlineStackProbe(
+ MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
+ // Collect PS_probed_stackalloc pseudos to expand. Collecting first avoids
+ // issues with modifying the block while iterating.
+ SmallVector<MachineInstr *, 2> ToReplace;
+ for (MachineInstr &MI : PrologueMBB)
+ if (MI.getOpcode() == Hexagon::PS_probed_stackalloc)
+ ToReplace.push_back(&MI);
+
+ auto &HST = MF.getSubtarget<HexagonSubtarget>();
+ auto &HII = *HST.getInstrInfo();
+ auto *TLI = HST.getTargetLowering();
+ Align StackAlign = getStackAlign();
+ unsigned ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
+ MachineInstr::MIFlag Flags = MachineInstr::FrameSetup;
+
+ for (MachineInstr *MI : ToReplace) {
+ MachineBasicBlock::iterator MBBI = MI->getIterator();
+ DebugLoc DL = PrologueMBB.findDebugLoc(MBBI);
+ Register TargetReg = MI->getOperand(0).getReg();
+
+ // Split the block: everything after the pseudo goes into ExitMBB.
+ MachineBasicBlock *MBB = MI->getParent();
+ MachineFunction::iterator InsertPt = std::next(MBB->getIterator());
+ MachineBasicBlock *LoopMBB =
+ MF.CreateMachineBasicBlock(MBB->getBasicBlock());
+ MF.insert(InsertPt, LoopMBB);
+ MachineBasicBlock *ExitMBB =
+ MF.CreateMachineBasicBlock(MBB->getBasicBlock());
+ MF.insert(InsertPt, ExitMBB);
+
+ // Move everything after the pseudo into ExitMBB.
+ ExitMBB->splice(ExitMBB->end(), MBB, std::next(MBBI), MBB->end());
+ ExitMBB->transferSuccessorsAndUpdatePHIs(MBB);
+
+ // LoopMBB: probe each page by decrementing SP and storing zero.
+ // When NumBytes is not an exact multiple of ProbeSize the loop
+ // will overshoot by up to ProbeSize-1 bytes; the final r29 = r28
+ // in ExitMBB corrects SP to the true target.
+ //
+ // The store is placed before the compare+branch so that the
+ // packetizer can bundle them into a single VLIW packet. All
+ // non-predicated instructions in a packet commit unconditionally,
+ // so the probe store executes on every iteration including the
+ // last (when the branch falls through).
+ //
+ // r29 = add(r29, #-ProbeSize)
+ // memw(r29+#0) = #0
+ // p0 = cmp.gtu(r29, r28)
+ // if (p0) jump LoopMBB
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::A2_addi),
+ Hexagon::R29)
+ .addReg(Hexagon::R29)
+ .addImm(-int(ProbeSize))
+ .setMIFlags(Flags);
+
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::S4_storeiri_io))
+ .addReg(Hexagon::R29)
+ .addImm(0)
+ .addImm(0)
+ .setMIFlags(Flags);
+
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::C2_cmpgtu),
+ Hexagon::P0)
+ .addReg(Hexagon::R29)
+ .addReg(TargetReg)
+ .setMIFlags(Flags);
+
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::J2_jumpt))
+ .addReg(Hexagon::P0)
+ .addMBB(LoopMBB)
+ .setMIFlags(Flags);
+
+ // ExitMBB: set final SP.
+ BuildMI(*ExitMBB, ExitMBB->begin(), DL, HII.get(Hexagon::A2_tfr),
+ Hexagon::R29)
+ .addReg(TargetReg)
+ .setMIFlags(Flags);
+
+ // Set up CFG edges.
+ MBB->addSuccessor(LoopMBB);
+ LoopMBB->addSuccessor(LoopMBB);
+ LoopMBB->addSuccessor(ExitMBB);
+
+ // Remove the pseudo.
+ MI->eraseFromParent();
+
+ // Recompute live-ins for the new blocks.
+ fullyRecomputeLiveIns({ExitMBB, LoopMBB});
+ }
+}
+
void HexagonFrameLowering::updateEntryPaths(MachineFunction &MF,
MachineBasicBlock &SaveB) const {
SetVector<unsigned> Worklist;
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h
index 926aadb01f50e..285e6a5896461 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h
@@ -113,6 +113,9 @@ class HexagonFrameLowering : public TargetFrameLowering {
void insertCFIInstructions(MachineFunction &MF) const;
+ void inlineStackProbe(MachineFunction &MF,
+ MachineBasicBlock &PrologueMBB) const override;
+
protected:
bool hasFPImpl(const MachineFunction &MF) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index acafd754969d7..6b5e5ef1a3c81 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -4045,3 +4045,21 @@ bool HexagonTargetLowering::isUsedByReturnOnly(SDNode *N,
Chain = Copy->getOperand(0);
return true;
}
+
+bool HexagonTargetLowering::hasInlineStackProbe(
+ const MachineFunction &MF) const {
+ if (MF.getFunction().hasFnAttribute("probe-stack"))
+ return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
+ "inline-asm";
+ return false;
+}
+
+unsigned HexagonTargetLowering::getStackProbeSize(const MachineFunction &MF,
+ Align StackAlign) const {
+ const Function &Fn = MF.getFunction();
+ unsigned StackProbeSize =
+ Fn.getFnAttributeAsParsedInteger("stack-probe-size", 4096);
+ // Round down to the stack alignment.
+ StackProbeSize = alignDown(StackProbeSize, StackAlign.value());
+ return StackProbeSize ? StackProbeSize : StackAlign.value();
+}
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 87041e696f0f4..53aff87091775 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -297,6 +297,9 @@ class HexagonTargetLowering : public TargetLowering {
MachineBasicBlock::instr_iterator &MBBI,
const TargetInstrInfo *TII) const override;
+ bool hasInlineStackProbe(const MachineFunction &MF) const override;
+ unsigned getStackProbeSize(const MachineFunction &MF, Align StackAlign) const;
+
private:
void initializeHVXLowering();
unsigned getPreferredHvxVectorAction(MVT VecTy) const;
diff --git a/llvm/lib/Target/Hexagon/HexagonPseudo.td b/llvm/lib/Target/Hexagon/HexagonPseudo.td
index d75a5c0e51ce8..8736d4f05c321 100644
--- a/llvm/lib/Target/Hexagon/HexagonPseudo.td
+++ b/llvm/lib/Target/Hexagon/HexagonPseudo.td
@@ -335,6 +335,12 @@ let Defs = [R29], hasSideEffects = 1 in
def PS_alloca: Pseudo <(outs IntRegs:$Rd),
(ins IntRegs:$Rs, u32_0Imm:$A), "", []>;
+// Probed stack allocation pseudo. Expanded in inlineStackProbe() into a
+// compare-and-branch loop that touches each page.
+let Defs = [R29, P0], Uses = [R29], hasSideEffects = 1, mayStore = 1,
+ isCodeGenOnly = 1 in
+def PS_probed_stackalloc : Pseudo<(outs), (ins IntRegs:$target), "", []>;
+
// Load predicate.
let isExtendable = 1, opExtendable = 2, isExtentSigned = 1, opExtentBits = 13,
isCodeGenOnly = 1, isPseudo = 1, hasSideEffects = 0 in
diff --git a/llvm/test/CodeGen/Hexagon/stack-probing.ll b/llvm/test/CodeGen/Hexagon/stack-probing.ll
new file mode 100644
index 0000000000000..a59556a782a53
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/stack-probing.ll
@@ -0,0 +1,166 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+;; Small frame (< probe size): no probing loop, normal allocframe.
+; CHECK-LABEL: small_frame:
+; CHECK: allocframe(r29,#128):raw
+; CHECK-NOT: cmp.gtu
+; CHECK: dealloc_return
+define void @small_frame() #0 {
+entry:
+ %a = alloca [128 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; Large frame (> probe size): probing loop emitted.
+; CHECK-LABEL: large_frame:
+; CHECK: allocframe(r29,#0):raw
+; CHECK: r28 = add(r29,#-8192)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: r29 = add(r29,#-4096)
+; CHECK: p0 = cmp.gtu(r29,r28)
+; CHECK: if (p0.new) jump:t .LBB
+; CHECK: memw(r29+#0) = #0
+; CHECK: r29 = r28
+define void @large_frame() #0 {
+entry:
+ %a = alloca [8192 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; Exact multiple of probe size: probing loop still emitted.
+; CHECK-LABEL: exact_multiple:
+; CHECK: allocframe(r29,#0):raw
+; CHECK: r28 = add(r29,#-12288)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: r29 = add(r29,#-4096)
+; CHECK: p0 = cmp.gtu(r29,r28)
+; CHECK: if (p0.new) jump:t .LBB
+; CHECK: memw(r29+#0) = #0
+; CHECK: r29 = r28
+define void @exact_multiple() #0 {
+entry:
+ %a = alloca [12288 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; No frame pointer path with no call: probing works without allocframe.
+; CHECK-LABEL: no_fp_large:
+; CHECK-NOT: allocframe
+; CHECK: r28 = add(r29,#-8192)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: r29 = add(r29,#-4096)
+; CHECK: p0 = cmp.gtu(r29,r28)
+; CHECK: if (p0.new) jump:t .LBB
+; CHECK: memw(r29+#0) = #0
+; CHECK: r29 = r28
+define void @no_fp_large() #1 {
+entry:
+ %a = alloca [8192 x i8], align 1
+ store volatile i8 0, ptr %a
+ ret void
+}
+
+;; Custom probe size of 512 bytes.
+; CHECK-LABEL: custom_probe_size:
+; CHECK: allocframe(r29,#0):raw
+; CHECK: r28 = add(r29,#-8192)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: r29 = add(r29,#-512)
+; CHECK: p0 = cmp.gtu(r29,r28)
+; CHECK: if (p0.new) jump:t .LBB
+; CHECK: memw(r29+#0) = #0
+; CHECK: r29 = r28
+define void @custom_probe_size() #2 {
+entry:
+ %a = alloca [8192 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; No probe attribute: normal codegen, no probing.
+; CHECK-LABEL: no_probe:
+; CHECK: allocframe
+; CHECK-NOT: cmp.gtu
+; CHECK: dealloc_return
+define void @no_probe() {
+entry:
+ %a = alloca [8192 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; Frame >= ALLOCFRAME_MAX (16384): allocframe(#0) + probed alloc.
+; CHECK-LABEL: very_large_frame:
+; CHECK: allocframe(r29,#0):raw
+; CHECK: r28 = add(r29,#-20480)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: r29 = add(r29,#-4096)
+; CHECK: p0 = cmp.gtu(r29,r28)
+; CHECK: if (p0.new) jump:t .LBB
+; CHECK: memw(r29+#0) = #0
+; CHECK: r29 = r28
+define void @very_large_frame() #0 {
+entry:
+ %a = alloca [20480 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; Frame == probe size exactly: no probing loop, normal allocframe.
+; CHECK-LABEL: exact_probe_size:
+; CHECK: allocframe(r29,#4096):raw
+; CHECK-NOT: cmp.gtu
+; CHECK: dealloc_return
+define void @exact_probe_size() #0 {
+entry:
+ %a = alloca [4096 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; Large frame requiring constant-extended immediate (> 32767).
+; CHECK-LABEL: const_extd_frame:
+; CHECK: allocframe(r29,#0):raw
+; CHECK: r28 = add(r29,##-65536)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: r29 = add(r29,#-4096)
+; CHECK: p0 = cmp.gtu(r29,r28)
+; CHECK: if (p0.new) jump:t .LBB
+; CHECK: memw(r29+#0) = #0
+; CHECK: r29 = r28
+define void @const_extd_frame() #0 {
+entry:
+ %a = alloca [65536 x i8], align 1
+ call void @use(ptr %a)
+ ret void
+}
+
+;; Callee-saved register spills coexist with probing.
+; CHECK-LABEL: callee_saved_regs:
+; CHECK: allocframe(r29,#0):raw
+; CHECK: r28 = add(r29,#-8216)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: r29 = add(r29,#-4096)
+; CHECK: p0 = cmp.gtu(r29,r28)
+; CHECK: if (p0.new) jump:t .LBB
+; CHECK: memw(r29+#0) = #0
+; CHECK: r29 = r28
+; CHECK: memd(r29+##{{[0-9]+}}) = r{{[0-9]+}}:{{[0-9]+}}
+; CHECK: dealloc_return
+define void @callee_saved_regs(ptr %p) #0 {
+entry:
+ %a = alloca [8192 x i8], align 1
+ call void @use(ptr %a)
+ call void asm sideeffect "", "~{r16},~{r17},~{r18},~{r19}"()
+ call void @use(ptr %p)
+ ret void
+}
+
+declare void @use(ptr)
+
+attributes #0 = { "probe-stack"="inline-asm" }
+attributes #1 = { nounwind "frame-pointer"="none" "probe-stack"="inline-asm" }
+attributes #2 = { "probe-stack"="inline-asm" "stack-probe-size"="512" }
>From 2eeed596ce8c215d38108eccd91530cb505c0a92 Mon Sep 17 00:00:00 2001
From: Brian Cain <brian.cain at oss.qualcomm.com>
Date: Mon, 13 Apr 2026 15:52:43 -0700
Subject: [PATCH 2/3] fixup! [Hexagon] Add probe-stack=inline-asm support for
stack clash protection
fixup! [Hexagon] Add probe-stack=inline-asm support for stack clash protection
Check VLIW packet boundaries in probe loop to verify the store is
bundled with the compare+branch.
---
llvm/test/CodeGen/Hexagon/stack-probing.ll | 84 ++++++++++++++--------
1 file changed, 56 insertions(+), 28 deletions(-)
diff --git a/llvm/test/CodeGen/Hexagon/stack-probing.ll b/llvm/test/CodeGen/Hexagon/stack-probing.ll
index a59556a782a53..1f5d506939a72 100644
--- a/llvm/test/CodeGen/Hexagon/stack-probing.ll
+++ b/llvm/test/CodeGen/Hexagon/stack-probing.ll
@@ -17,10 +17,14 @@ entry:
; CHECK: allocframe(r29,#0):raw
; CHECK: r28 = add(r29,#-8192)
; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
-; CHECK: r29 = add(r29,#-4096)
-; CHECK: p0 = cmp.gtu(r29,r28)
-; CHECK: if (p0.new) jump:t .LBB
-; CHECK: memw(r29+#0) = #0
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
; CHECK: r29 = r28
define void @large_frame() #0 {
entry:
@@ -34,10 +38,14 @@ entry:
; CHECK: allocframe(r29,#0):raw
; CHECK: r28 = add(r29,#-12288)
; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
-; CHECK: r29 = add(r29,#-4096)
-; CHECK: p0 = cmp.gtu(r29,r28)
-; CHECK: if (p0.new) jump:t .LBB
-; CHECK: memw(r29+#0) = #0
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
; CHECK: r29 = r28
define void @exact_multiple() #0 {
entry:
@@ -51,10 +59,14 @@ entry:
; CHECK-NOT: allocframe
; CHECK: r28 = add(r29,#-8192)
; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
-; CHECK: r29 = add(r29,#-4096)
-; CHECK: p0 = cmp.gtu(r29,r28)
-; CHECK: if (p0.new) jump:t .LBB
-; CHECK: memw(r29+#0) = #0
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
; CHECK: r29 = r28
define void @no_fp_large() #1 {
entry:
@@ -68,10 +80,14 @@ entry:
; CHECK: allocframe(r29,#0):raw
; CHECK: r28 = add(r29,#-8192)
; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
-; CHECK: r29 = add(r29,#-512)
-; CHECK: p0 = cmp.gtu(r29,r28)
-; CHECK: if (p0.new) jump:t .LBB
-; CHECK: memw(r29+#0) = #0
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-512)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
; CHECK: r29 = r28
define void @custom_probe_size() #2 {
entry:
@@ -97,10 +113,14 @@ entry:
; CHECK: allocframe(r29,#0):raw
; CHECK: r28 = add(r29,#-20480)
; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
-; CHECK: r29 = add(r29,#-4096)
-; CHECK: p0 = cmp.gtu(r29,r28)
-; CHECK: if (p0.new) jump:t .LBB
-; CHECK: memw(r29+#0) = #0
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
; CHECK: r29 = r28
define void @very_large_frame() #0 {
entry:
@@ -126,10 +146,14 @@ entry:
; CHECK: allocframe(r29,#0):raw
; CHECK: r28 = add(r29,##-65536)
; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
-; CHECK: r29 = add(r29,#-4096)
-; CHECK: p0 = cmp.gtu(r29,r28)
-; CHECK: if (p0.new) jump:t .LBB
-; CHECK: memw(r29+#0) = #0
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
; CHECK: r29 = r28
define void @const_extd_frame() #0 {
entry:
@@ -143,10 +167,14 @@ entry:
; CHECK: allocframe(r29,#0):raw
; CHECK: r28 = add(r29,#-8216)
; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
-; CHECK: r29 = add(r29,#-4096)
-; CHECK: p0 = cmp.gtu(r29,r28)
-; CHECK: if (p0.new) jump:t .LBB
-; CHECK: memw(r29+#0) = #0
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
; CHECK: r29 = r28
; CHECK: memd(r29+##{{[0-9]+}}) = r{{[0-9]+}}:{{[0-9]+}}
; CHECK: dealloc_return
>From cfbea163fe5a851982e324c08ee8c4187a0b082f Mon Sep 17 00:00:00 2001
From: Brian Cain <brian.cain at oss.qualcomm.com>
Date: Tue, 23 Jun 2026 21:54:57 -0700
Subject: [PATCH 3/3] fixup! [Hexagon] Add probe-stack=inline-asm support for
stack clash protection
Extend expandAlloca() to emit a probe loop for variable-length array
(dynamic alloca) allocations when probe-stack=inline-asm is active.
The dynamic allocation size is a runtime value so the probe loop is
always emitted. The sequence is:
Rd = sub(r29, Rs) ; compute target SP
[Rd = and(Rd, #-A)] ; align if necessary
LoopMBB:
r29 = add(r29, #-ProbeSize)
memw(r29+#0) = #0
p0 = cmp.gtu(r29, Rd)
if (p0.new) jump:t LoopMBB
ExitMBB:
r29 = Rd ; snap SP to exact target
[Rd = add(Rd, #CF)] ; call-frame adjustment
Rd holds the aligned target SP throughout the loop so the
final r29 = Rd correctly snaps SP even when Rs is not a multiple of
ProbeSize.
---
.../Target/Hexagon/HexagonFrameLowering.cpp | 180 ++++++++++++++----
.../lib/Target/Hexagon/HexagonFrameLowering.h | 5 +-
llvm/test/CodeGen/Hexagon/stack-probing.ll | 93 +++++++++
3 files changed, 237 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
index 2bd40d0eb6f28..3fb8a89363553 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -614,7 +614,7 @@ void HexagonFrameLowering::insertPrologueInBlock(MachineBasicBlock &MBB,
for (auto *MI : AdjustRegs) {
assert((MI->getOpcode() == Hexagon::PS_alloca) && "Expected alloca");
- expandAlloca(MI, HII, SP, MaxCF);
+ expandAlloca(MI, MF, HII, SP, MaxCF);
MI->eraseFromParent();
}
@@ -2629,62 +2629,164 @@ void HexagonFrameLowering::optimizeSpillSlots(MachineFunction &MF,
}
}
-void HexagonFrameLowering::expandAlloca(MachineInstr *AI,
- const HexagonInstrInfo &HII, Register SP, unsigned CF) const {
+void HexagonFrameLowering::expandAlloca(MachineInstr *AI, MachineFunction &MF,
+ const HexagonInstrInfo &HII,
+ Register SP, unsigned CF) const {
MachineBasicBlock &MB = *AI->getParent();
DebugLoc DL = AI->getDebugLoc();
unsigned A = AI->getOperand(2).getImm();
- // Have
- // Rd = alloca Rs, #A
- //
- // If Rs and Rd are different registers, use this sequence:
- // Rd = sub(r29, Rs)
- // r29 = sub(r29, Rs)
- // Rd = and(Rd, #-A) ; if necessary
- // r29 = and(r29, #-A) ; if necessary
- // Rd = add(Rd, #CF) ; CF size aligned to at most A
- // otherwise, do
- // Rd = sub(r29, Rs)
- // Rd = and(Rd, #-A) ; if necessary
- // r29 = Rd
- // Rd = add(Rd, #CF) ; CF size aligned to at most A
-
MachineOperand &RdOp = AI->getOperand(0);
MachineOperand &RsOp = AI->getOperand(1);
Register Rd = RdOp.getReg(), Rs = RsOp.getReg();
+ auto &HST = MF.getSubtarget<HexagonSubtarget>();
+ auto *TLI = HST.getTargetLowering();
+ bool NeedsProbing = TLI->hasInlineStackProbe(MF);
+
+ if (!NeedsProbing) {
+ // Have
+ // Rd = alloca Rs, #A
+ //
+ // If Rs and Rd are different registers, use this sequence:
+ // Rd = sub(r29, Rs)
+ // r29 = sub(r29, Rs)
+ // Rd = and(Rd, #-A) ; if necessary
+ // r29 = and(r29, #-A) ; if necessary
+ // Rd = add(Rd, #CF) ; CF size aligned to at most A
+ // otherwise, do
+ // Rd = sub(r29, Rs)
+ // Rd = and(Rd, #-A) ; if necessary
+ // r29 = Rd
+ // Rd = add(Rd, #CF) ; CF size aligned to at most A
+
+ // Rd = sub(r29, Rs)
+ BuildMI(MB, AI, DL, HII.get(Hexagon::A2_sub), Rd).addReg(SP).addReg(Rs);
+ if (Rs != Rd) {
+ // r29 = sub(r29, Rs)
+ BuildMI(MB, AI, DL, HII.get(Hexagon::A2_sub), SP).addReg(SP).addReg(Rs);
+ }
+ if (A > 8) {
+ // Rd = and(Rd, #-A)
+ BuildMI(MB, AI, DL, HII.get(Hexagon::A2_andir), Rd)
+ .addReg(Rd)
+ .addImm(-int64_t(A));
+ if (Rs != Rd)
+ BuildMI(MB, AI, DL, HII.get(Hexagon::A2_andir), SP)
+ .addReg(SP)
+ .addImm(-int64_t(A));
+ }
+ if (Rs == Rd) {
+ // r29 = Rd
+ BuildMI(MB, AI, DL, HII.get(TargetOpcode::COPY), SP).addReg(Rd);
+ }
+ if (CF > 0) {
+ // Rd = add(Rd, #CF)
+ BuildMI(MB, AI, DL, HII.get(Hexagon::A2_addi), Rd).addReg(Rd).addImm(CF);
+ }
+ return;
+ }
+
+ // Stack probing for dynamic allocation. The size Rs is a runtime value
+ // so the probe loop is always emitted; it is a no-op when Rs is small.
+ //
+ // Compute the target SP into Rd (with optional alignment), then probe
+ // each page on the way down:
+ //
+ // Rd = sub(r29, Rs)
+ // [Rd = and(Rd, #-A)] ; if alignment > 8
+ // LoopMBB:
+ // r29 = add(r29, #-ProbeSize)
+ // memw(r29+#0) = #0
+ // p0 = cmp.gtu(r29, Rd)
+ // if (p0.new) jump:t LoopMBB
+ // ExitMBB:
+ // r29 = Rd
+ // [Rd = add(Rd, #CF)] ; if CF > 0
+ // <rest of original block>
+ //
+ // Rd holds the exact (aligned) target SP throughout the loop, so the
+ // final "r29 = Rd" snaps SP to the correct value even when Rs is not
+ // a multiple of ProbeSize.
+
+ Align StackAlign = getStackAlign();
+ unsigned ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
+ MachineInstr::MIFlag Flags = MachineInstr::FrameSetup;
+
+ // Emit target-SP computation into Rd before splitting the block.
// Rd = sub(r29, Rs)
BuildMI(MB, AI, DL, HII.get(Hexagon::A2_sub), Rd)
.addReg(SP)
- .addReg(Rs);
- if (Rs != Rd) {
- // r29 = sub(r29, Rs)
- BuildMI(MB, AI, DL, HII.get(Hexagon::A2_sub), SP)
- .addReg(SP)
- .addReg(Rs);
- }
+ .addReg(Rs)
+ .setMIFlags(Flags);
if (A > 8) {
- // Rd = and(Rd, #-A)
+ // Rd = and(Rd, #-A)
BuildMI(MB, AI, DL, HII.get(Hexagon::A2_andir), Rd)
.addReg(Rd)
- .addImm(-int64_t(A));
- if (Rs != Rd)
- BuildMI(MB, AI, DL, HII.get(Hexagon::A2_andir), SP)
- .addReg(SP)
- .addImm(-int64_t(A));
- }
- if (Rs == Rd) {
- // r29 = Rd
- BuildMI(MB, AI, DL, HII.get(TargetOpcode::COPY), SP)
- .addReg(Rd);
+ .addImm(-int64_t(A))
+ .setMIFlags(Flags);
}
+
+ // Split the block: everything after AI goes into ExitMBB.
+ MachineFunction::iterator InsertPt = std::next(MB.getIterator());
+ MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(MB.getBasicBlock());
+ MF.insert(InsertPt, LoopMBB);
+ MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(MB.getBasicBlock());
+ MF.insert(InsertPt, ExitMBB);
+
+ // Move instructions after AI (exclusive) into ExitMBB.
+ ExitMBB->splice(ExitMBB->end(), &MB, std::next(AI->getIterator()), MB.end());
+ ExitMBB->transferSuccessorsAndUpdatePHIs(&MB);
+
+ // LoopMBB: probe each page.
+ // r29 = add(r29, #-ProbeSize)
+ // memw(r29+#0) = #0
+ // p0 = cmp.gtu(r29, Rd)
+ // if (p0.new) jump:t LoopMBB
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::A2_addi), Hexagon::R29)
+ .addReg(Hexagon::R29)
+ .addImm(-int(ProbeSize))
+ .setMIFlags(Flags);
+
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::S4_storeiri_io))
+ .addReg(Hexagon::R29)
+ .addImm(0)
+ .addImm(0)
+ .setMIFlags(Flags);
+
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::C2_cmpgtu),
+ Hexagon::P0)
+ .addReg(Hexagon::R29)
+ .addReg(Rd)
+ .setMIFlags(Flags);
+
+ BuildMI(*LoopMBB, LoopMBB->end(), DL, HII.get(Hexagon::J2_jumpt))
+ .addReg(Hexagon::P0)
+ .addMBB(LoopMBB)
+ .setMIFlags(Flags);
+
+ // ExitMBB: snap SP to exact target, then apply CF offset to Rd.
+ // r29 = Rd
+ // [Rd = add(Rd, #CF)]
+ MachineBasicBlock::iterator ExitIt = ExitMBB->begin();
+ BuildMI(*ExitMBB, ExitIt, DL, HII.get(Hexagon::A2_tfr), Hexagon::R29)
+ .addReg(Rd)
+ .setMIFlags(Flags);
if (CF > 0) {
- // Rd = add(Rd, #CF)
- BuildMI(MB, AI, DL, HII.get(Hexagon::A2_addi), Rd)
+ BuildMI(*ExitMBB, ExitIt, DL, HII.get(Hexagon::A2_addi), Rd)
.addReg(Rd)
- .addImm(CF);
+ .addImm(CF)
+ .setMIFlags(Flags);
}
+
+ // Wire up CFG edges.
+ MB.addSuccessor(LoopMBB);
+ LoopMBB->addSuccessor(LoopMBB);
+ LoopMBB->addSuccessor(ExitMBB);
+
+ // Recompute live-ins for the new blocks. AI is still in MB at this
+ // point; the caller erases it after expandAlloca returns.
+ fullyRecomputeLiveIns({ExitMBB, LoopMBB});
}
bool HexagonFrameLowering::needsAligna(const MachineFunction &MF) const {
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h
index 285e6a5896461..2e876ab034cbe 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h
@@ -122,8 +122,9 @@ class HexagonFrameLowering : public TargetFrameLowering {
private:
using CSIVect = std::vector<CalleeSavedInfo>;
- void expandAlloca(MachineInstr *AI, const HexagonInstrInfo &TII,
- Register SP, unsigned CF) const;
+ void expandAlloca(MachineInstr *AI, MachineFunction &MF,
+ const HexagonInstrInfo &TII, Register SP,
+ unsigned CF) const;
void insertPrologueInBlock(MachineBasicBlock &MBB, bool PrologueStubs) const;
void insertEpilogueInBlock(MachineBasicBlock &MBB) const;
void insertAllocframe(MachineBasicBlock &MBB,
diff --git a/llvm/test/CodeGen/Hexagon/stack-probing.ll b/llvm/test/CodeGen/Hexagon/stack-probing.ll
index 1f5d506939a72..74c8161833c95 100644
--- a/llvm/test/CodeGen/Hexagon/stack-probing.ll
+++ b/llvm/test/CodeGen/Hexagon/stack-probing.ll
@@ -187,7 +187,100 @@ entry:
ret void
}
+;; VLA (dynamic alloca): probe loop emitted for runtime-sized allocation.
+; CHECK-LABEL: vla_basic:
+; CHECK: allocframe
+; CHECK: r[[TGT:[0-9]+]] = sub(r29,r{{[0-9]+}})
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r[[TGT]])
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
+; CHECK: r29 = r[[TGT]]
+define void @vla_basic(i32 %n) #0 {
+entry:
+ %sz = zext i32 %n to i64
+ %buf = alloca i8, i64 %sz, align 1
+ call void @use(ptr %buf)
+ ret void
+}
+
+;; VLA without probe attribute: plain sub(r29,Rs), no probe loop.
+; CHECK-LABEL: vla_no_probe:
+; CHECK: sub(r29,r
+; CHECK-NOT: cmp.gtu
+; CHECK: dealloc_return
+define void @vla_no_probe(i32 %n) {
+entry:
+ %sz = zext i32 %n to i64
+ %buf = alloca i8, i64 %sz, align 1
+ call void @use(ptr %buf)
+ ret void
+}
+
+;; Static large frame + VLA: two separate probe loops are emitted.
+; CHECK-LABEL: vla_and_static:
+; CHECK: allocframe(r29,#0):raw
+; CHECK: r28 = add(r29,#-8200)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r28)
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
+; CHECK: r29 = r28
+; CHECK: r[[TGT2:[0-9]+]] = sub(r29,r{{[0-9]+}})
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r[[TGT2]])
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
+; CHECK: r29 = r[[TGT2]]
+define void @vla_and_static(i32 %n) #0 {
+entry:
+ %static = alloca [8192 x i8], align 1
+ %sz = zext i32 %n to i64
+ %dyn = alloca i8, i64 %sz, align 1
+ call void @use2(ptr %static, ptr %dyn)
+ ret void
+}
+
+;; VLA with alignment > 8: target pointer is aligned before probing.
+; CHECK-LABEL: vla_align16:
+; CHECK: allocframe
+; CHECK: r[[TGT3:[0-9]+]] = sub(r29,r{{[0-9]+}})
+; CHECK: r[[TGT3]] = and(r[[TGT3]],#-16)
+; CHECK: .LBB{{[0-9]+}}_{{[0-9]+}}:
+; CHECK: {
+; CHECK-NEXT: r29 = add(r29,#-4096)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r29,r[[TGT3]])
+; CHECK-NEXT: if (p0.new) jump:t .LBB
+; CHECK-NEXT: memw(r29+#0) = #0
+; CHECK-NEXT: }
+; CHECK: r29 = r[[TGT3]]
+define void @vla_align16(i32 %n) #0 {
+entry:
+ %sz = zext i32 %n to i64
+ %buf = alloca i8, i64 %sz, align 16
+ call void @use(ptr %buf)
+ ret void
+}
+
declare void @use(ptr)
+declare void @use2(ptr, ptr)
attributes #0 = { "probe-stack"="inline-asm" }
attributes #1 = { nounwind "frame-pointer"="none" "probe-stack"="inline-asm" }
More information about the llvm-commits
mailing list