[llvm-branch-commits] [llvm] [AMDGPU] Lower FREEZE of i1 in SILowerI1Copies (PR #240475)

Mitch Briles via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Mon Nov 9 13:47:00 PST 2026


https://github.com/MitchBriles updated https://github.com/llvm/llvm-project/pull/240475

>From 2c502a1d02888fc9402287e6392c329e5684e6f3 Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 6 Nov 2026 20:21:49 -0700
Subject: [PATCH] [AMDGPU] Lower FREEZE of i1 in SILowerI1Copies

A FREEZE of a vreg_1 was left as is, which fails the verifier. Freeze
the lane mask, and lower the rest like a copy.

Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
 llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp    |  18 ++-
 .../CodeGen/AMDGPU/lower-i1-copies-freeze.mir | 143 ++++++++++++++++++
 2 files changed, 158 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir

diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
index 4fd9576f18c5ab..90ab1bce3fa6f1 100644
--- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
@@ -17,7 +17,7 @@
 // pass.
 //
 // The only instructions that use or define vreg_1 virtual registers are COPY,
-// PHI, and IMPLICIT_DEF.
+// FREEZE, PHI, and IMPLICIT_DEF.
 //
 //===----------------------------------------------------------------------===//
 
@@ -591,7 +591,7 @@ bool Vreg1LoweringHelper::lowerCopiesToI1() {
 
     for (MachineInstr &MI : MBB) {
       if (MI.getOpcode() != AMDGPU::IMPLICIT_DEF &&
-          MI.getOpcode() != AMDGPU::COPY)
+          MI.getOpcode() != AMDGPU::COPY && MI.getOpcode() != AMDGPU::FREEZE)
         continue;
 
       Register DstReg = MI.getOperand(0).getReg();
@@ -630,6 +630,16 @@ bool Vreg1LoweringHelper::lowerCopiesToI1() {
         MI.getOperand(1).setIsKill(false);
       }
 
+      // Freeze the lane mask, and lower the rest like a copy.
+      if (MI.getOpcode() == AMDGPU::FREEZE) {
+        Register FrozenReg = AMDGPU::createLaneMaskReg(MRI, LaneMaskRegAttrs);
+        BuildMI(MBB, MI, DL, TII->get(AMDGPU::FREEZE), FrozenReg)
+            .addReg(SrcReg);
+        MI.setDesc(TII->get(AMDGPU::COPY));
+        MI.getOperand(1).setReg(FrozenReg);
+        SrcReg = FrozenReg;
+      }
+
       // Defs in a loop that are observed outside the loop must be transformed
       // into appropriate bit manipulation.
       std::vector<MachineBasicBlock *> DomBlocks = {&MBB};
@@ -770,7 +780,9 @@ void Vreg1LoweringHelper::collectIncomingValuesFromPhi(
     } else if (IncomingDef->getOpcode() == AMDGPU::IMPLICIT_DEF) {
       continue;
     } else {
-      assert(IncomingDef->isPHI() || PhiRegisters.count(IncomingReg));
+      assert(IncomingDef->isPHI() ||
+             IncomingDef->getOpcode() == AMDGPU::FREEZE ||
+             PhiRegisters.count(IncomingReg));
     }
 
     Incomings.emplace_back(IncomingReg, IncomingMBB, Register());
diff --git a/llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir b/llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir
new file mode 100644
index 00000000000000..28025117aef84e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-i1-copies-freeze.mir
@@ -0,0 +1,143 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu11.00 -run-pass=si-i1-copies -o - %s | FileCheck %s
+
+---
+name: freeze_phi
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: freeze_phi
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_LT_U32_e64 [[COPY]], [[COPY1]], implicit $exec
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF [[V_CMP_LT_U32_e64_]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY]], [[COPY1]], implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sreg_32 = PHI [[DEF1]], %bb.0, [[V_CMP_EQ_U32_e64_]], %bb.1
+  ; CHECK-NEXT:   SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[PHI]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY [[FREEZE]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY2]]
+  ; CHECK-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY]], 0, [[COPY1]], [[COPY3]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
+  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: $vgpr0, $vgpr1
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:sreg_32 = V_CMP_LT_U32_e64 %0, %1, implicit $exec
+    %3:sreg_32 = IMPLICIT_DEF
+    %4:sreg_32 = SI_IF %2, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2
+
+    %5:sreg_32 = V_CMP_EQ_U32_e64 %0, %1, implicit $exec
+    %6:vreg_1 = COPY %5
+
+  bb.2:
+    %7:vreg_1 = PHI %3, %bb.0, %6, %bb.1
+    SI_END_CF %4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    %8:vreg_1 = FREEZE %7
+    %9:sreg_32_xm0_xexec = COPY %8
+    %10:vgpr_32 = V_CNDMASK_B32_e64 0, %0, 0, %1, %9, implicit $exec
+    $vgpr0 = COPY %10
+    SI_RETURN implicit $vgpr0
+...
+---
+name: freeze_in_loop
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: freeze_in_loop
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sreg_32 = PHI [[DEF1]], %bb.0, %18, %bb.1
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:sreg_32 = PHI [[DEF]], %bb.0, %14, %bb.1
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:sreg_32 = PHI [[S_MOV_B32_]], %bb.0, %8, %bb.1
+  ; CHECK-NEXT:   [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[V_MOV_B32_e32_]], %bb.0, %10, %bb.1
+  ; CHECK-NEXT:   [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[PHI1]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[FREEZE]]
+  ; CHECK-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[PHI3]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY3]]
+  ; CHECK-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[COPY4]], implicit $exec
+  ; CHECK-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[PHI3]], [[V_CNDMASK_B32_e64_]], 0, implicit $exec
+  ; CHECK-NEXT:   [[V_CMP_GE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_GE_U32_e64 [[V_ADD_U32_e64_]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   [[SI_IF_BREAK:%[0-9]+]]:sreg_32 = SI_IF_BREAK [[V_CMP_GE_U32_e64_]], [[PHI2]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32 = S_ANDN2_B32 [[PHI]], $exec_lo, implicit-def dead $scc
+  ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY3]], $exec_lo, implicit-def dead $scc
+  ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_ANDN2_B32_]], [[S_AND_B32_]], implicit-def dead $scc
+  ; CHECK-NEXT:   SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI4:%[0-9]+]]:sreg_32 = PHI [[SI_IF_BREAK]], %bb.1
+  ; CHECK-NEXT:   SI_END_CF [[PHI4]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY2]], 0, [[COPY1]], [[COPY5]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY [[V_CNDMASK_B32_e64_1]]
+  ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
+  bb.0:
+    successors: %bb.1
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = COPY $vgpr2
+    %3:sreg_32 = IMPLICIT_DEF
+    %4:sreg_32 = S_MOV_B32 0
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %6:vreg_1 = COPY %3
+
+  bb.1:
+    successors: %bb.2, %bb.1
+
+    %7:sreg_32 = PHI %4, %bb.0, %16, %bb.1
+    %8:vgpr_32 = PHI %5, %bb.0, %14, %bb.1
+    %9:vreg_1 = PHI %6, %bb.0, %11, %bb.1
+    %10:vreg_1 = FREEZE %9
+    %12:sreg_32 = V_CMP_EQ_U32_e64 %8, %0, implicit $exec
+    %11:vreg_1 = COPY %12
+    %13:sreg_32_xm0_xexec = COPY %10
+    %15:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %13, implicit $exec
+    %14:vgpr_32 = V_ADD_U32_e64 %8, %15, 0, implicit $exec
+    %17:sreg_32 = V_CMP_GE_U32_e64 %14, %0, implicit $exec
+    %16:sreg_32 = SI_IF_BREAK %17, %7, implicit-def dead $scc
+    SI_LOOP %16, %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    %18:vreg_1 = PHI %10, %bb.1
+    %19:sreg_32 = PHI %16, %bb.1
+    SI_END_CF %19, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    %20:sreg_32_xm0_xexec = COPY %18
+    %21:vgpr_32 = V_CNDMASK_B32_e64 0, %2, 0, %1, %20, implicit $exec
+    $vgpr0 = COPY %21
+    SI_RETURN_TO_EPILOG $vgpr0
+...



More information about the llvm-branch-commits mailing list