[llvm] [AMDGPU] Model structured control-flow lane masks as i1 (PR #209158)

via llvm-commits llvm-commits at lists.llvm.org
Mon Jul 13 05:17:53 PDT 2026 

llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-ir

@llvm/pr-subscribers-llvm-globalisel

Author: Pankaj Dwivedi (PankajDwivedi-25)

<details>
<summary>Changes</summary>

Ref: #<!-- -->206435.

The lane mask from the structured CF intrinsics (`if`, `else`, `if.break`, `loop`, `end.cf`) is a per-lane value, but it was typed as an overloaded `i32`/`i64`. 

This patch attempts to retype these intrinsics to `i1`, the natural type for a wave-width lane mask.

---

Patch is 1.49 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/209158.diff


72 Files Affected:

- (modified) llvm/include/llvm/IR/IntrinsicsAMDGPU.td (+7-7) 
- (modified) llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp (+1-1) 
- (modified) llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp (+5-5) 
- (modified) llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h (+6-8) 
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp (+7-8) 
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp (+6-1) 
- (modified) llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td (+4-1) 
- (modified) llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp (+8-15) 
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+2-2) 
- (modified) llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll (+40-40) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll (+286-80) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll (+286-80) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll (+41-28) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll (+131-96) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll (+92-60) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll (+115-74) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll (+24-16) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll (+18-12) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll (+36-18) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll (+20-20) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll (+2-2) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll (+7-3) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i64.ll (+6-6) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll (+14-7) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll (+17-12) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll (+64-46) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir (+214-155) 
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir (+322-223) 
- (modified) llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll (+37-29) 
- (modified) llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll (+95-35) 
- (modified) llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll (+830-286) 
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll (+122-113) 
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll (+112-103) 
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll (+112-103) 
- (modified) llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll (+67-28) 
- (modified) llvm/test/CodeGen/AMDGPU/div_i128.ll (+36-32) 
- (modified) llvm/test/CodeGen/AMDGPU/div_v2i128.ll (+188-188) 
- (modified) llvm/test/CodeGen/AMDGPU/diverge-switch-default.ll (+3-3) 
- (modified) llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll (+4-4) 
- (added) llvm/test/CodeGen/AMDGPU/divergent-loop-i1-lane-mask.ll (+118) 
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll (+945-395) 
- (modified) llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll (+2-2) 
- (modified) llvm/test/CodeGen/AMDGPU/literal64.ll (+57-24) 
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll (+23-12) 
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll (+2092-689) 
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll (+2092-689) 
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll (+2350-610) 
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll (+2350-610) 
- (modified) llvm/test/CodeGen/AMDGPU/loop-on-function-argument.ll (+4-4) 
- (modified) llvm/test/CodeGen/AMDGPU/loop_break.ll (+24-24) 
- (modified) llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll (+1-1) 
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir (+6-6) 
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir (+5-5) 
- (modified) llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll (+306-153) 
- (modified) llvm/test/CodeGen/AMDGPU/memset-pattern.ll (+715-299) 
- (modified) llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll (+45-45) 
- (modified) llvm/test/CodeGen/AMDGPU/multilevel-break.ll (+16-16) 
- (modified) llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll (+28-30) 
- (modified) llvm/test/CodeGen/AMDGPU/rem_i128.ll (+18-16) 
- (modified) llvm/test/CodeGen/AMDGPU/sdiv64.ll (+5-5) 
- (modified) llvm/test/CodeGen/AMDGPU/si-annotate-cf-unreachable.ll (+1-1) 
- (modified) llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll (+15-15) 
- (added) llvm/test/CodeGen/AMDGPU/si-annotate-loop-i1-lane-mask.ll (+43) 
- (modified) llvm/test/CodeGen/AMDGPU/si-annotatecfg-multiple-backedges.ll (+9-9) 
- (modified) llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll (+23-23) 
- (modified) llvm/test/CodeGen/AMDGPU/srem64.ll (+5-5) 
- (modified) llvm/test/CodeGen/AMDGPU/udiv64.ll (+5-5) 
- (modified) llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll (+21-21) 
- (modified) llvm/test/CodeGen/AMDGPU/urem64.ll (+4-4) 
- (modified) llvm/test/CodeGen/AMDGPU/valu-i1.ll (+11-11) 
- (modified) llvm/test/CodeGen/AMDGPU/wave32.ll (+5-5) 
- (modified) llvm/test/CodeGen/MIR/AMDGPU/syncscopes.mir (+5-5) 


``````````diff
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 21882e247c027..12c96d354760b 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -3903,24 +3903,24 @@ def int_amdgcn_cooperative_atomic_store_8x16B : AMDGPUCooperativeAtomicStore<llv
 // convergence control tokens. The corresponding MIR pseudos are marked as
 // having side effects, which is sufficient to prevent optimizations without
 // having to mark them as convergent.
-def int_amdgcn_if : Intrinsic<[llvm_i1_ty, llvm_anyint_ty],
+def int_amdgcn_if : Intrinsic<[llvm_i1_ty, llvm_i1_ty],
   [llvm_i1_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
 >;
 
-def int_amdgcn_else : Intrinsic<[llvm_i1_ty, llvm_anyint_ty],
-  [llvm_anyint_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
+def int_amdgcn_else : Intrinsic<[llvm_i1_ty, llvm_i1_ty],
+  [llvm_i1_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
 >;
 
-def int_amdgcn_if_break : Intrinsic<[llvm_anyint_ty],
-  [llvm_i1_ty, LLVMMatchType<0>],
+def int_amdgcn_if_break : Intrinsic<[llvm_i1_ty],
+  [llvm_i1_ty, llvm_i1_ty],
   [IntrNoMem, IntrWillReturn, IntrNoCallback, IntrNoFree]
 >;
 
 def int_amdgcn_loop : Intrinsic<[llvm_i1_ty],
-  [llvm_anyint_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
+  [llvm_i1_ty], [IntrWillReturn, IntrNoCallback, IntrNoFree]
 >;
 
-def int_amdgcn_end_cf : Intrinsic<[], [llvm_anyint_ty],
+def int_amdgcn_end_cf : Intrinsic<[], [llvm_i1_ty],
   [IntrWillReturn, IntrNoCallback, IntrNoFree]>;
 
 // Represent unreachable in a divergent region.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
index cbb4269e17260..2ae63025d2fd7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
@@ -208,7 +208,7 @@ bool DivergenceLoweringHelper::lowerTemporalDivergence() {
 
   for (auto [Reg, UseInst, _] : MUI->getTemporalDivergenceList()) {
     if (MRI->getType(Reg) == LLT::scalar(1) || MUI->isDivergentAtDef(Reg) ||
-        ILMA.isS32S64LaneMask(Reg))
+        ILMA.isLaneMask(Reg))
       continue;
 
     Register CachedTDCopy = TDCache.lookup(Reg);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index f36935d8c0e8f..0b7b2fe8cb13b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -83,8 +83,8 @@ IntrinsicLaneMaskAnalyzer::IntrinsicLaneMaskAnalyzer(MachineFunction &MF)
   initLaneMaskIntrinsics(MF);
 }
 
-bool IntrinsicLaneMaskAnalyzer::isS32S64LaneMask(Register Reg) const {
-  return S32S64LaneMask.contains(Reg);
+bool IntrinsicLaneMaskAnalyzer::isLaneMask(Register Reg) const {
+  return LaneMask.contains(Reg);
 }
 
 void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) {
@@ -92,13 +92,13 @@ void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) {
     for (auto &MI : MBB) {
       GIntrinsic *GI = dyn_cast<GIntrinsic>(&MI);
       if (GI && GI->is(Intrinsic::amdgcn_if_break)) {
-        S32S64LaneMask.insert(MI.getOperand(3).getReg());
-        S32S64LaneMask.insert(MI.getOperand(0).getReg());
+        LaneMask.insert(MI.getOperand(3).getReg());
+        LaneMask.insert(MI.getOperand(0).getReg());
       }
 
       if (MI.getOpcode() == AMDGPU::SI_IF ||
           MI.getOpcode() == AMDGPU::SI_ELSE) {
-        S32S64LaneMask.insert(MI.getOperand(0).getReg());
+        LaneMask.insert(MI.getOperand(0).getReg());
       }
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
index 5e1000ee0ab26..40e7be7f7bbd7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
@@ -31,19 +31,17 @@ getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
                           GISelValueTracking *ValueTracking = nullptr,
                           bool CheckNUW = false);
 
-// Currently finds S32/S64 lane masks that can be declared as divergent by
-// uniformity analysis (all are phis at the moment).
-// These are defined as i32/i64 in some IR intrinsics (not as i1).
-// Tablegen forces(via telling that lane mask IR intrinsics are uniform) most of
-// S32/S64 lane masks to be uniform, as this results in them ending up with sgpr
-// reg class after instruction-select, don't search for all of them.
+// Finds lane masks produced/consumed by the control flow intrinsics. These are
+// i1 values that live in wave-width lane mask registers (SReg_1). They are used
+// to assign such values to the Vcc (lane mask) register bank so that they
+// select to a wave mask register class.
 class IntrinsicLaneMaskAnalyzer {
-  SmallDenseSet<Register, 8> S32S64LaneMask;
+  SmallDenseSet<Register, 8> LaneMask;
   MachineRegisterInfo &MRI;
 
 public:
   IntrinsicLaneMaskAnalyzer(MachineFunction &MF);
-  bool isS32S64LaneMask(Register Reg) const;
+  bool isLaneMask(Register Reg) const;
 
 private:
   void initLaneMaskIntrinsics(MachineFunction &MF);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 7f7225ad6d311..acc71206e8cd3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1875,14 +1875,13 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
       .Any({{UniS1, _, S64}, {{UniInVcc}, {IntrId, Vgpr64, Vgpr32}}})
       .Any({{DivS1, _, S64}, {{Vcc}, {IntrId, Vgpr64, Vgpr32}}});
 
-  // This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.
-  addRulesForIOpcs({amdgcn_end_cf})
-      .Any({{_, UniS32}, {{}, {IntrId, Sgpr32}}})
-      .Any({{_, UniS64}, {{}, {IntrId, Sgpr64}}});
-
-  addRulesForIOpcs({amdgcn_if_break}, Standard)
-      .Uni(S64, {{Sgpr64}, {IntrId, Vcc, Sgpr64}})
-      .Uni(S32, {{Sgpr32}, {IntrId, Vcc, Sgpr32}});
+  // The "intrinsic lane mask" is a divergent i1 value living in a wave-width
+  // lane mask register (the Vcc bank / SReg_1). Modeling it as a divergent i1
+  // lets the existing divergent-i1-phi lowering thread the "phi.broken" mask
+  // through loops as a lane mask.
+  addRulesForIOpcs({amdgcn_end_cf}).Any({{_, S1}, {{}, {IntrId, Vcc}}});
+
+  addRulesForIOpcs({amdgcn_if_break}).Any({{S1}, {{Vcc}, {IntrId, Vcc, Vcc}}});
 
   addRulesForIOpcs({amdgcn_exp})
       .Any({{_, _, _, S32, S32, S32, S32},
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
index 247522a0dd43b..dc6bfa9e83bda 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
@@ -110,8 +110,13 @@ class RegBankSelectHelper {
   }
 
   const RegisterBank *getRegBankToAssign(Register Reg) {
+    // An "intrinsic lane mask" is an s1 value that lives in a wave-width lane
+    // mask register. Assign it to the Vcc (lane mask) bank rather than a scalar
+    // sgpr boolean so that it selects to a wave mask register class.
+    if (ILMA.isLaneMask(Reg) && MRI.getType(Reg) == LLT::scalar(1))
+      return VccRB;
     if (!isTemporalDivergenceCopy(Reg) &&
-        (MUI.isUniformAtDef(Reg) || ILMA.isS32S64LaneMask(Reg)))
+        (MUI.isUniformAtDef(Reg) || ILMA.isLaneMask(Reg)))
       return SgprRB;
     if (MRI.getType(Reg) == LLT::scalar(1))
       return VccRB;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 2f7f72bd58f0e..df8a56ef51ead 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -376,6 +376,10 @@ def : SourceOfDivergence<int_amdgcn_ds_read_tr16_b64>;
 def : SourceOfDivergence<int_amdgcn_if>;
 def : SourceOfDivergence<int_amdgcn_else>;
 def : SourceOfDivergence<int_amdgcn_loop>;
+// The "intrinsic lane mask" result is an i1 value that lives in a wave-width
+// lane mask register (SReg_1), i.e. a divergent i1 from the analysis's point of
+// view. This lets the forwarding phis be treated as lane masks.
+def : SourceOfDivergence<int_amdgcn_if_break>;
 def : SourceOfDivergence<int_amdgcn_inverse_ballot>;
 
 foreach intr = AMDGPUImageDimAtomicIntrinsics in
@@ -401,7 +405,6 @@ def : AlwaysUniform<int_amdgcn_readlane>;
 def : AlwaysUniform<int_amdgcn_icmp>;
 def : AlwaysUniform<int_amdgcn_fcmp>;
 def : AlwaysUniform<int_amdgcn_ballot>;
-def : AlwaysUniform<int_amdgcn_if_break>;
 def : AlwaysUniform<int_amdgcn_cluster_workgroup_id_x>;
 def : AlwaysUniform<int_amdgcn_cluster_workgroup_id_y>;
 def : AlwaysUniform<int_amdgcn_cluster_workgroup_id_z>;
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index a7f98e79ce61d..09578918def17 100644
--- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -45,13 +45,10 @@ class SIAnnotateControlFlow {
 
   Type *Boolean;
   Type *Void;
-  Type *IntMask;
-  Type *ReturnStruct;
 
   ConstantInt *BoolTrue;
   ConstantInt *BoolFalse;
   PoisonValue *BoolPoison;
-  Constant *IntMaskZero;
 
   Function *If = nullptr;
   Function *Else = nullptr;
@@ -115,14 +112,10 @@ void SIAnnotateControlFlow::initialize(const GCNSubtarget &ST) {
 
   Void = Type::getVoidTy(Context);
   Boolean = Type::getInt1Ty(Context);
-  IntMask = ST.isWave32() ? Type::getInt32Ty(Context)
-                           : Type::getInt64Ty(Context);
-  ReturnStruct = StructType::get(Boolean, IntMask);
 
   BoolTrue = ConstantInt::getTrue(Context);
   BoolFalse = ConstantInt::getFalse(Context);
   BoolPoison = PoisonValue::get(Boolean);
-  IntMaskZero = ConstantInt::get(IntMask, 0);
 }
 
 /// Is the branch condition uniform or did the StructurizeCFG pass
@@ -188,7 +181,7 @@ bool SIAnnotateControlFlow::openIf(CondBrInst *Term) {
     return false;
 
   IRBuilder<> IRB(Term);
-  Value *IfCall = IRB.CreateCall(getDecl(If, Intrinsic::amdgcn_if, IntMask),
+  Value *IfCall = IRB.CreateCall(getDecl(If, Intrinsic::amdgcn_if, {}),
                                  {Term->getCondition()});
   Value *Cond = IRB.CreateExtractValue(IfCall, {0});
   Value *Mask = IRB.CreateExtractValue(IfCall, {1});
@@ -204,8 +197,8 @@ bool SIAnnotateControlFlow::insertElse(CondBrInst *Term) {
   }
 
   IRBuilder<> IRB(Term);
-  Value *ElseCall = IRB.CreateCall(
-      getDecl(Else, Intrinsic::amdgcn_else, {IntMask, IntMask}), {popSaved()});
+  Value *ElseCall =
+      IRB.CreateCall(getDecl(Else, Intrinsic::amdgcn_else, {}), {popSaved()});
   Value *Cond = IRB.CreateExtractValue(ElseCall, {0});
   Value *Mask = IRB.CreateExtractValue(ElseCall, {1});
   Term->setCondition(Cond);
@@ -220,7 +213,7 @@ Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken,
 
   auto CreateBreak = [this, Cond, Broken](Instruction *I) -> CallInst * {
     return IRBuilder<>(I).CreateCall(
-        getDecl(IfBreak, Intrinsic::amdgcn_if_break, IntMask), {Cond, Broken});
+        getDecl(IfBreak, Intrinsic::amdgcn_if_break, {}), {Cond, Broken});
   };
 
   if (Instruction *Inst = dyn_cast<Instruction>(Cond)) {
@@ -267,7 +260,7 @@ bool SIAnnotateControlFlow::handleLoop(CondBrInst *Term) {
     return false;
 
   BasicBlock *Target = Term->getSuccessor(1);
-  PHINode *Broken = PHINode::Create(IntMask, 0, "phi.broken");
+  PHINode *Broken = PHINode::Create(Boolean, 0, "phi.broken");
   Broken->insertBefore(Target->begin());
 
   Value *Cond = Term->getCondition();
@@ -275,7 +268,7 @@ bool SIAnnotateControlFlow::handleLoop(CondBrInst *Term) {
   Value *Arg = handleLoopCondition(Cond, Broken, L, Term);
 
   for (BasicBlock *Pred : predecessors(Target)) {
-    Value *PHIValue = IntMaskZero;
+    Value *PHIValue = BoolFalse;
     if (Pred == BB) // Remember the value of the previous iteration.
       PHIValue = Arg;
     // If the backedge from Pred to Target could be executed before the exit
@@ -287,7 +280,7 @@ bool SIAnnotateControlFlow::handleLoop(CondBrInst *Term) {
   }
 
   CallInst *LoopCall = IRBuilder<>(Term).CreateCall(
-      getDecl(Loop, Intrinsic::amdgcn_loop, IntMask), {Arg});
+      getDecl(Loop, Intrinsic::amdgcn_loop, {}), {Arg});
   Term->setCondition(LoopCall);
 
   push(Term->getSuccessor(0), Arg);
@@ -332,7 +325,7 @@ bool SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) {
     // condition, for now just avoid copying these DebugLocs so that stepping
     // out of the then/else block in a debugger doesn't step to the condition.
     IRB.SetCurrentDebugLocation(DebugLoc());
-    IRB.CreateCall(getDecl(EndCf, Intrinsic::amdgcn_end_cf, IntMask), {Exec});
+    IRB.CreateCall(getDecl(EndCf, Intrinsic::amdgcn_end_cf, {}), {Exec});
   }
 
   return true;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9cf33b4ccbb87..e11d321bd83b3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20648,8 +20648,8 @@ static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
   // FIXME: We assume we never cast the mask results of a control flow
   // intrinsic.
   // Early exit if the type won't be consistent as a compile time hack.
-  IntegerType *IT = dyn_cast<IntegerType>(V->getType());
-  if (!IT || IT->getBitWidth() != WaveSize)
+  // The mask results of the control flow intrinsics are i1 lane masks.
+  if (!V->getType()->isIntegerTy(1))
     return false;
 
   if (!isa<Instruction>(V))
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll
index b92daa64040e4..a3c3b5f544ad6 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/control-flow-intrinsics.ll
@@ -1,102 +1,102 @@
 ; RUN: opt -mtriple amdgcn-mesa-mesa3d -passes='print<uniformity>' -disable-output %s 2>&1 | FileCheck %s
 
-; Tests control flow intrinsics that should be treated as uniform
+; Tests uniformity of the AMDGPU control flow intrinsics. Their lane-mask
+; operands and results are i1 values that are divergent (per-lane).
 
 ; CHECK: for function 'test_if_break':
 ; CHECK: DIVERGENT: %cond = icmp eq i32 %arg0, 0
-; CHECK-NOT: DIVERGENT
-; CHECK: ret void
-define amdgpu_ps void @test_if_break(i32 %arg0, i64 inreg %saved) {
+; CHECK: DIVERGENT: %break = call i1 @llvm.amdgcn.if.break(i1 %cond, i1 %saved)
+define amdgpu_ps void @test_if_break(i32 %arg0, i1 inreg %saved) {
 entry:
   %cond = icmp eq i32 %arg0, 0
-  %break = call i64 @llvm.amdgcn.if.break.i64.i64(i1 %cond, i64 %saved)
-  store volatile i64 %break, ptr addrspace(1) undef
+  %break = call i1 @llvm.amdgcn.if.break(i1 %cond, i1 %saved)
+  store volatile i1 %break, ptr addrspace(1) undef
   ret void
 }
 
 ; CHECK: for function 'test_if':
 ; CHECK: DIVERGENT: %cond = icmp eq i32 %arg0, 0
-; CHECK-NEXT: DIVERGENT: %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
-; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i64 } %if, 0
+; CHECK-NEXT: DIVERGENT: %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i1 } %if, 0
 ; CHECK-NOT: DIVERGENT
 ; CHECK: DIVERGENT: %if.bool.ext = zext i1 %if.bool to i32
 define void @test_if(i32 %arg0) {
 entry:
   %cond = icmp eq i32 %arg0, 0
-  %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
-  %if.bool = extractvalue { i1, i64 } %if, 0
-  %if.mask = extractvalue { i1, i64 } %if, 1
+  %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+  %if.bool = extractvalue { i1, i1 } %if, 0
+  %if.mask = extractvalue { i1, i1 } %if, 1
   %if.bool.ext = zext i1 %if.bool to i32
   store volatile i32 %if.bool.ext, ptr addrspace(1) undef
-  store volatile i64 %if.mask, ptr addrspace(1) undef
+  store volatile i1 %if.mask, ptr addrspace(1) undef
   ret void
 }
 
 ; The result should still be treated as divergent, even with a uniform source.
 ; CHECK: for function 'test_if_uniform':
 ; CHECK-NOT: DIVERGENT
-; CHECK: DIVERGENT: %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
-; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i64 } %if, 0
+; CHECK: DIVERGENT: %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i1 } %if, 0
 ; CHECK-NOT: DIVERGENT
 ; CHECK: DIVERGENT: %if.bool.ext = zext i1 %if.bool to i32
 define amdgpu_ps void @test_if_uniform(i32 inreg %arg0) {
 entry:
   %cond = icmp eq i32 %arg0, 0
-  %if = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %cond)
-  %if.bool = extractvalue { i1, i64 } %if, 0
-  %if.mask = extractvalue { i1, i64 } %if, 1
+  %if = call { i1, i1 } @llvm.amdgcn.if(i1 %cond)
+  %if.bool = extractvalue { i1, i1 } %if, 0
+  %if.mask = extractvalue { i1, i1 } %if, 1
   %if.bool.ext = zext i1 %if.bool to i32
   store volatile i32 %if.bool.ext, ptr addrspace(1) undef
-  store volatile i64 %if.mask, ptr addrspace(1) undef
+  store volatile i1 %if.mask, ptr addrspace(1) undef
   ret void
 }
 
 ; CHECK: for function 'test_loop_uniform':
-; CHECK: DIVERGENT: %loop = call i1 @llvm.amdgcn.loop.i64(i64 %mask)
-define amdgpu_ps void @test_loop_uniform(i64 inreg %mask) {
+; CHECK: DIVERGENT: %loop = call i1 @llvm.amdgcn.loop(i1 %mask)
+define amdgpu_ps void @test_loop_uniform(i1 inreg %mask) {
 entry:
-  %loop = call i1 @llvm.amdgcn.loop.i64(i64 %mask)
+  %loop = call i1 @llvm.amdgcn.loop(i1 %mask)
   %loop.ext = zext i1 %loop to i32
   store volatile i32 %loop.ext, ptr addrspace(1) undef
   ret void
 }
 
 ; CHECK: for function 'test_else':
-; CHECK: DIVERGENT: %else = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
-; CHECK: DIVERGENT:       %else.bool = extractvalue { i1, i64 } %else, 0
-; CHECK: {{^[ \t]+}}%else.mask = extractvalue { i1, i64 } %else, 1
-define amdgpu_ps void @test_else(i64 inreg %mask) {
+; CHECK: DIVERGENT: %else = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+; CHECK: DIVERGENT:       %else.bool = extractvalue { i1, i1 } %else, 0
+; CHECK: {{^[ \t]+}}%else.mask = extractvalue { i1, i1 } %else, 1
+define amdgpu_ps void @test_else(i1 inreg %mask) {
 entry:
-  %else = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
-  %else.bool = extractvalue { i1, i64 } %else, 0
-  %else.mask = extractvalue { i1, i64 } %else, 1
+  %else = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+  %else.bool = extractvalue { i1, i1 } %else, 0
+  %else.mask = extractvalue { i1, i1 } %else, 1
   %else.bool.ext = zext i1 %else.bool to i32
   store volatile i32 %else.bool.ext, ptr addrspace(1) undef
-  store volatile i64 %else.mask, ptr addrspace(1) undef
+  store volatile i1 %else.mask, ptr addrspace(1) undef
   ret void
 }
 
 ; This case is probably always broken
 ; CHECK: for function 'test_else_divergent_mask':
-; CHECK: DIVERGENT: %if = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
-; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i64 } %if, 0
+; CHECK: DIVERGENT: %if = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+; CHECK-NEXT: DIVERGENT: %if.bool = extractvalue { i1, i1 } %if, 0
 ; CHECK-NOT: DIVERGENT
 ; CHECK: DIVERGENT: %if.bool.ext = zext i1 %if.bool to i32
-define void @test_else_divergent_mask(i64 %mask) {
+define void @test_else_divergent_mask(i1 %mask) {
 entry:
-  %if = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %mask)
-  %if.bool = extractvalue { i1, i64 } %if, 0
-  %if.mask = extractvalue { i1, i64 } %if, 1
+  %if = call { i1, i1 } @llvm.amdgcn.else(i1 %mask)
+  %if.bool = extractvalue { i1, i1 } %if, 0
+  %if.mask = extractvalue { i1, i1 } %if, 1
   %if.bool.ext = zext i1 %if.bool to i32
   store volatile i32 %if.bool.ext, ptr addrspace(1) undef
-  store volatile i64 %if.mask, ptr addrspace(1) undef
+  store volatile i1 %if.mask, ptr addrspace(1) undef
   ret void
 }
 
-declare { i1, i64 } @llvm.amdgcn.if.i64(i1) #0
-declare { i1, i64 } @llvm.amdgcn.else.i64.i64(i64) #0
-declare i64 @llvm.amdgcn.if.break.i64.i64(i1, i64) #1
-declare i1 @llvm.amdgcn.loop.i64(i64) #1
+declare { i1, i1 } @llvm.amdgcn.if(i1) #0
+declare { i1, i1 } @llvm.amdgcn.else(i1) #0
+declare i1 @llvm.amdgcn.if.break(i1, i1) #1
+declare i1 @llvm.amdgcn.loop(i1) #1
 
 attributes #0 = { convergent nounwind }
 attributes #1 = { convergent nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll b/llvm/test/CodeGen...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/209158

More information about the llvm-commits mailing list