[llvm-branch-commits] [llvm] [AMDGPU] Lower FREEZE of i1 in SILowerI1Copies (PR #240475)
Mitch Briles via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 10 01:54:01 PST 2026
https://github.com/MitchBriles updated https://github.com/llvm/llvm-project/pull/240475
>From 1ddf6f026059d1a28f35a1584abd51058931c7fc Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 6 Nov 2026 20:21:49 -0700
Subject: [PATCH] [AMDGPU] Lower FREEZE of i1 in SILowerI1Copies
A FREEZE of a vreg_1 was left as is, which fails the verifier. Freeze
the lane mask, and lower the rest like a copy.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp | 18 ++-
.../CodeGen/AMDGPU/lower-i1-copies-freeze.mir | 143 ++++++++++++++++++
2 files changed, 158 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir
diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
index 4fd9576f18c5ab..90ab1bce3fa6f1 100644
--- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
@@ -17,7 +17,7 @@
// pass.
//
// The only instructions that use or define vreg_1 virtual registers are COPY,
-// PHI, and IMPLICIT_DEF.
+// FREEZE, PHI, and IMPLICIT_DEF.
//
//===----------------------------------------------------------------------===//
@@ -591,7 +591,7 @@ bool Vreg1LoweringHelper::lowerCopiesToI1() {
for (MachineInstr &MI : MBB) {
if (MI.getOpcode() != AMDGPU::IMPLICIT_DEF &&
- MI.getOpcode() != AMDGPU::COPY)
+ MI.getOpcode() != AMDGPU::COPY && MI.getOpcode() != AMDGPU::FREEZE)
continue;
Register DstReg = MI.getOperand(0).getReg();
@@ -630,6 +630,16 @@ bool Vreg1LoweringHelper::lowerCopiesToI1() {
MI.getOperand(1).setIsKill(false);
}
+ // Freeze the lane mask, and lower the rest like a copy.
+ if (MI.getOpcode() == AMDGPU::FREEZE) {
+ Register FrozenReg = AMDGPU::createLaneMaskReg(MRI, LaneMaskRegAttrs);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::FREEZE), FrozenReg)
+ .addReg(SrcReg);
+ MI.setDesc(TII->get(AMDGPU::COPY));
+ MI.getOperand(1).setReg(FrozenReg);
+ SrcReg = FrozenReg;
+ }
+
// Defs in a loop that are observed outside the loop must be transformed
// into appropriate bit manipulation.
std::vector<MachineBasicBlock *> DomBlocks = {&MBB};
@@ -770,7 +780,9 @@ void Vreg1LoweringHelper::collectIncomingValuesFromPhi(
} else if (IncomingDef->getOpcode() == AMDGPU::IMPLICIT_DEF) {
continue;
} else {
- assert(IncomingDef->isPHI() || PhiRegisters.count(IncomingReg));
+ assert(IncomingDef->isPHI() ||
+ IncomingDef->getOpcode() == AMDGPU::FREEZE ||
+ PhiRegisters.count(IncomingReg));
}
Incomings.emplace_back(IncomingReg, IncomingMBB, Register());
diff --git a/llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir b/llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir
new file mode 100644
index 00000000000000..28025117aef84e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir
@@ -0,0 +1,143 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu11.00 -run-pass=si-i1-copies -o - %s | FileCheck %s
+
+---
+name: freeze_phi
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: freeze_phi
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_LT_U32_e64 [[COPY]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[V_CMP_LT_U32_e64_]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32 = PHI [[DEF1]], %bb.0, [[V_CMP_EQ_U32_e64_]], %bb.1
+ ; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[PHI]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[FREEZE]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY2]]
+ ; CHECK-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY]], 0, [[COPY1]], [[COPY3]], implicit $exec
+ ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $vgpr1
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:sreg_32 = V_CMP_LT_U32_e64 %0, %1, implicit $exec
+ %3:sreg_32 = IMPLICIT_DEF
+ %4:sreg_32 = SI_IF %2, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+
+ %5:sreg_32 = V_CMP_EQ_U32_e64 %0, %1, implicit $exec
+ %6:vreg_1 = COPY %5
+
+ bb.2:
+ %7:vreg_1 = PHI %3, %bb.0, %6, %bb.1
+ SI_END_CF %4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ %8:vreg_1 = FREEZE %7
+ %9:sreg_32_xm0_xexec = COPY %8
+ %10:vgpr_32 = V_CNDMASK_B32_e64 0, %0, 0, %1, %9, implicit $exec
+ $vgpr0 = COPY %10
+ SI_RETURN implicit $vgpr0
+...
+---
+name: freeze_in_loop
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: freeze_in_loop
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32 = PHI [[DEF1]], %bb.0, %18, %bb.1
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:sreg_32 = PHI [[DEF]], %bb.0, %14, %bb.1
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:sreg_32 = PHI [[S_MOV_B32_]], %bb.0, %8, %bb.1
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[V_MOV_B32_e32_]], %bb.0, %10, %bb.1
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[PHI1]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[FREEZE]]
+ ; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[PHI3]], [[COPY]], implicit $exec
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY3]]
+ ; CHECK-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[COPY4]], implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[PHI3]], [[V_CNDMASK_B32_e64_]], 0, implicit $exec
+ ; CHECK-NEXT: [[V_CMP_GE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_GE_U32_e64 [[V_ADD_U32_e64_]], [[COPY]], implicit $exec
+ ; CHECK-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_32 = SI_IF_BREAK [[V_CMP_GE_U32_e64_]], [[PHI2]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32 = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def dead $scc
+ ; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY3]], $exec_lo, implicit-def dead $scc
+ ; CHECK-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_ANDN2_B32_]], [[S_AND_B32_]], implicit-def dead $scc
+ ; CHECK-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:sreg_32 = PHI [[SI_IF_BREAK]], %bb.1
+ ; CHECK-NEXT: SI_END_CF [[PHI4]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[S_OR_B32_]]
+ ; CHECK-NEXT: [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY2]], 0, [[COPY1]], [[COPY5]], implicit $exec
+ ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_e64_1]]
+ ; CHECK-NEXT: SI_RETURN_TO_EPILOG $vgpr0
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ %3:sreg_32 = IMPLICIT_DEF
+ %4:sreg_32 = S_MOV_B32 0
+ %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %6:vreg_1 = COPY %3
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %7:sreg_32 = PHI %4, %bb.0, %16, %bb.1
+ %8:vgpr_32 = PHI %5, %bb.0, %14, %bb.1
+ %9:vreg_1 = PHI %6, %bb.0, %11, %bb.1
+ %10:vreg_1 = FREEZE %9
+ %12:sreg_32 = V_CMP_EQ_U32_e64 %8, %0, implicit $exec
+ %11:vreg_1 = COPY %12
+ %13:sreg_32_xm0_xexec = COPY %10
+ %15:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %13, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e64 %8, %15, 0, implicit $exec
+ %17:sreg_32 = V_CMP_GE_U32_e64 %14, %0, implicit $exec
+ %16:sreg_32 = SI_IF_BREAK %17, %7, implicit-def dead $scc
+ SI_LOOP %16, %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ S_BRANCH %bb.2
+
+ bb.2:
+ %18:vreg_1 = PHI %10, %bb.1
+ %19:sreg_32 = PHI %16, %bb.1
+ SI_END_CF %19, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ %20:sreg_32_xm0_xexec = COPY %18
+ %21:vgpr_32 = V_CNDMASK_B32_e64 0, %2, 0, %1, %20, implicit $exec
+ $vgpr0 = COPY %21
+ SI_RETURN_TO_EPILOG $vgpr0
+...
More information about the llvm-branch-commits
mailing list