[llvm] DAG: Materialize a directly-lowered gc.relocate in its own block (PR #222894)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 02:17:44 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-selectiondag
Author: Matt Arsenault (arsenm)
<details>
<summary>Changes</summary>
Fixes machine verifier errors with -early-live-intervals.
A NoRelocate statepoint value is a directly-lowered leaf value. When its
gc.relocate is in another block, it was made available by exporting it from the
statepoint's block. For an invoke this defines the vreg after the call, which
does not dominate the use reached along the unwind edge, giving an invalid live
range (or a silent read of an undefined register without -verify-machineinstrs).
Since the leaf has no chain or operands, rematerialize it at the gc.relocate
instead of exporting it. undef already rematerializes there. A gc.relocate
value is always a pointer, so only the integer-constant and frame-index leaves
are handled.
Co-authored-by: Claude (Claude-Opus-4.8)
---
Full diff: https://github.com/llvm/llvm-project/pull/222894.diff
3 Files Affected:
- (modified) llvm/include/llvm/CodeGen/FunctionLoweringInfo.h (+25)
- (modified) llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp (+46-6)
- (added) llvm/test/CodeGen/X86/statepoint-relocate-eh.ll (+89)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
index bc7c4deef1bb1..9e580181123ad 100644
--- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -14,6 +14,7 @@
#ifndef LLVM_CODEGEN_FUNCTIONLOWERINGINFO_H
#define LLVM_CODEGEN_FUNCTIONLOWERINGINFO_H
+#include "llvm/ADT/APInt.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/IndexedMap.h"
@@ -27,6 +28,7 @@
#include "llvm/IR/Value.h"
#include "llvm/Support/KnownBits.h"
#include <cassert>
+#include <optional>
#include <utility>
#include <vector>
@@ -41,8 +43,11 @@ class MachineFunction;
class MachineInstr;
class MachineRegisterInfo;
class MVT;
+class SDLoc;
+class SDValue;
class SelectionDAG;
class TargetLowering;
+struct EVT;
template <typename T> class GenericSSAContext;
using SSAContext = GenericSSAContext<Function>;
@@ -94,6 +99,22 @@ class FunctionLoweringInfo {
/// Track virtual registers created for exception pointers.
DenseMap<const Value *, Register> CatchPadExceptionPointers;
+ /// A directly-lowered statepoint value (see willLowerDirectly): a leaf that
+ /// can be rebuilt at a gc.relocate in another block.
+ struct StatepointDirectLeaf {
+ enum LeafKind { FrameIndex, Constant };
+ LeafKind Kind;
+ APInt IntValue; // Constant: the integer value.
+ int FrameIndexValue; // FrameIndex: the frame index.
+
+ /// Capture the leaf \p V, which must be a non-undef directly-lowered value.
+ LLVM_ABI explicit StatepointDirectLeaf(SDValue V);
+
+ /// Rebuild the captured leaf as a fresh SDValue of type \p VT.
+ LLVM_ABI SDValue rematerialize(SelectionDAG &DAG, const SDLoc &DL,
+ EVT VT) const;
+ };
+
/// Helper object to track which of three possible relocation mechanisms are
/// used for a particular value being relocated over a statepoint.
struct StatepointRelocationRecord {
@@ -117,6 +138,10 @@ class FunctionLoweringInfo {
int FI;
Register Reg;
} payload;
+
+ // For a NoRelocate value whose gc.relocate is in another block, this holds
+ // enough to re-materialize the directly-lowered leaf at the gc.relocate.
+ std::optional<StatepointDirectLeaf> RematLeaf;
};
/// Keep track of each value which was relocated and the strategy used to
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index d66d6f5a99416..d84be7ef1c0ce 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -260,6 +260,29 @@ static bool willLowerDirectly(SDValue Incoming) {
return isIntOrFPConstant(Incoming) || Incoming.isUndef();
}
+FunctionLoweringInfo::StatepointDirectLeaf::StatepointDirectLeaf(SDValue V) {
+ assert(willLowerDirectly(V) && !V.isUndef() &&
+ "not a non-undef directly-lowered leaf");
+ if (auto *FI = dyn_cast<FrameIndexSDNode>(V)) {
+ Kind = FrameIndex;
+ FrameIndexValue = FI->getIndex();
+ } else {
+ Kind = Constant;
+ IntValue = cast<ConstantSDNode>(V)->getAPIntValue();
+ }
+}
+
+SDValue FunctionLoweringInfo::StatepointDirectLeaf::rematerialize(
+ SelectionDAG &DAG, const SDLoc &DL, EVT VT) const {
+ switch (Kind) {
+ case FrameIndex:
+ return DAG.getFrameIndex(FrameIndexValue, VT);
+ case Constant:
+ return DAG.getConstant(IntValue, DL, VT);
+ }
+ llvm_unreachable("unhandled directly-lowered leaf kind");
+}
+
/// Try to find existing copies of the incoming values in stack slots used for
/// statepoint spilling. If we can find a spill slot for the incoming value,
/// mark that slot as allocated, and reuse the same slot for this safepoint.
@@ -931,12 +954,18 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
Record.payload.FI = cast<FrameIndexSDNode>(Loc)->getIndex();
} else {
Record.type = RecordType::NoRelocate;
- // If we didn't relocate a value, we'll essentialy end up inserting an
- // additional use of the original value when lowering the gc.relocate.
- // We need to make sure the value is available at the new use, which
- // might be in another block.
- if (Relocate->getParent() != StatepointInstr->getParent())
- ExportFromCurrentBlock(V);
+ // This value lowers directly, so it is a leaf that can be rematerialized.
+ assert(willLowerDirectly(SDV) &&
+ "NoRelocate value is expected to lower directly");
+
+ // A gc.relocate in another block must rebuild the value there. Exporting
+ // it would define a vreg after the statepoint call that does not dominate
+ // a use reached along the unwind (EH pad) edge. Record the leaf for
+ // visitGCRelocate; a local gc.relocate can reference the value directly,
+ // and undef re-lowers trivially, so neither is recorded.
+ if (Relocate->getParent() != StatepointInstr->getParent() &&
+ !SDV.isUndef())
+ Record.RematLeaf.emplace(SDV);
}
RelocationMap[Relocate] = Record;
}
@@ -1291,6 +1320,17 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
}
assert(Record.type == RecordType::NoRelocate);
+
+ // Rebuild a leaf recorded for a cross-block gc.relocate instead of using a
+ // value from the statepoint's block.
+ if (Record.RematLeaf) {
+ EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+ Relocate.getType());
+ setValue(&Relocate,
+ Record.RematLeaf->rematerialize(DAG, getCurSDLoc(), VT));
+ return;
+ }
+
SDValue SD = getValue(DerivedPtr);
if (SD.isUndef() && SD.getValueType().getSizeInBits() <= 64) {
diff --git a/llvm/test/CodeGen/X86/statepoint-relocate-eh.ll b/llvm/test/CodeGen/X86/statepoint-relocate-eh.ll
new file mode 100644
index 0000000000000..323d01c2e08c0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/statepoint-relocate-eh.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -early-live-intervals < %s | FileCheck %s
+
+; A gc.relocate of a directly-lowered value (a constant or an alloca)
+; in an invoke's landing pad must rematerialize the value at the
+; relocate, not export a vreg defined after the statepoint call in the
+; invoke block.
+
+target triple = "x86_64-pc-linux-gnu"
+
+declare void @foo()
+
+define ptr addrspace(1) @relocate_constant_in_landing_pad() gc "statepoint-example" personality ptr null {
+; CHECK-LABEL: relocate_constant_in_landing_pad:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .Ltmp0: # EH_LABEL
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: xorl %edi, %edi
+; CHECK-NEXT: callq *%rax
+; CHECK-NEXT: .Ltmp3:
+; CHECK-NEXT: .Ltmp1: # EH_LABEL
+; CHECK-NEXT: # %bb.1: # %common.ret
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: popq %rcx
+; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: retq
+; CHECK-NEXT: .LBB0_2: # %exceptional_return
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .Ltmp2: # EH_LABEL
+; CHECK-NEXT: movl $1, %eax
+; CHECK-NEXT: popq %rcx
+; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: retq
+entry:
+ %c = inttoptr i64 1 to ptr addrspace(1)
+ %sp = invoke token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 0, i32 0, ptr elementtype(void (ptr addrspace(1))) null, i32 1, i32 0, ptr addrspace(1) null, i32 0, i32 0) [ "gc-live"(ptr addrspace(1) null, ptr addrspace(1) %c) ]
+ to label %common.ret unwind label %exceptional_return
+
+common.ret:
+ ret ptr addrspace(1) null
+
+exceptional_return:
+ %landing_pad = landingpad token
+ cleanup
+ %c.relocated = call coldcc ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token %landing_pad, i32 0, i32 1)
+ ret ptr addrspace(1) %c.relocated
+}
+
+define ptr addrspace(1) @relocate_alloca_in_landing_pad() gc "statepoint-example" personality ptr null {
+; CHECK-LABEL: relocate_alloca_in_landing_pad:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .Ltmp4: # EH_LABEL
+; CHECK-NEXT: callq foo at PLT
+; CHECK-NEXT: .Ltmp7:
+; CHECK-NEXT: .Ltmp5: # EH_LABEL
+; CHECK-NEXT: # %bb.1: # %normal
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: popq %rcx
+; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: retq
+; CHECK-NEXT: .LBB1_2: # %lpad
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .Ltmp6: # EH_LABEL
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: popq %rcx
+; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: retq
+entry:
+ %a = alloca i32
+ %ac = addrspacecast ptr %a to ptr addrspace(1)
+ %sp = invoke token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 0, i32 0, ptr elementtype(void ()) @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(ptr addrspace(1) %ac) ]
+ to label %normal unwind label %lpad
+
+normal:
+ ret ptr addrspace(1) null
+
+lpad:
+ %lp = landingpad token cleanup
+ %ac.rel = call coldcc ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token %lp, i32 0, i32 0)
+ ret ptr addrspace(1) %ac.rel
+}
+
+declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...)
+declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token, i32 immarg, i32 immarg)
``````````
</details>
https://github.com/llvm/llvm-project/pull/222894
More information about the llvm-commits
mailing list