[llvm] [AMDGPU] Combine redundant ballot intrinsic calls (PR #218357)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 21:12:22 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/218357
>From ff5f7b4dbabe19cda949b9772c81107f8669eccd Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Tue, 18 Aug 2026 23:08:37 -0400
Subject: [PATCH] [AMDGPU] Combine redundant ballot intrinsic calls
Suppose there is a loop where there is a call to @llvm.amdgcn.ballot,
which maps to an instruction involving the exec mask as an operand. This
instruction duplicates if the loop is unrolled. With a higher number of
unrolled iterations, the code bloats with such redundant instructions
with $exec as there is no middle-end/backend pass which could combine
such instructions in a uniform CFG.
This patch introduces a transform in AMDGPUUniformIntrinsicCombine to
combine redundant calls to @llvm.amdgcn.ballot, to mitigate this issue.
The approach is to walk over the dominator tree and collect all calls to
@llvm.amdgcn.ballot. Map the result type and condition to the calls, to
avoid combining calls of different kinds. Calls A and B can be combined
into A iff:
- A and B are identical
- A dominates B
- all paths from A to B are uniform and exec-invariant.
Co-authored by Claude Opus 5
---
.../AMDGPU/AMDGPUUniformIntrinsicCombine.cpp | 174 ++++++-
.../CodeGen/AMDGPU/redundant-ballot-reads.ll | 448 ++++++++++++++++++
2 files changed, 617 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/redundant-ballot-reads.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
index 6b68c25825d7b..6e9c225679024 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
@@ -16,16 +16,26 @@
/// uniformity. And every instruction that's downstream and cares about dynamic
/// uniformity must be convergent (and isel will introduce v_readfirstlane for
/// them if their operands can't be proven statically uniform).
+///
+/// The pass additionally performs a convergence-aware CSE of
+/// llvm.amdgcn.ballot. Because ballot is convergent, the generic CSE passes
+/// refuse to merge two calls that live in different basic blocks: the result
+/// implicitly depends on the set of currently active lanes (exec). Here we can
+/// do better, because uniformity analysis lets us prove that exec is unchanged
+/// between two identical calls, in which case the later one is redundant.
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
#include "GCNSubtarget.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstVisitor.h"
@@ -135,8 +145,153 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
return false;
}
+/// Maximum number of basic blocks inspected while proving that exec is
+/// invariant between two ballots. Keeps the walk below linear-per-pair in
+/// pathological CFGs.
+static constexpr unsigned MaxExecInvarianceBlocks = 100;
+
+/// Returns true if \p I may change exec, i.e. the set of lanes that are active
+/// when the following instructions execute.
+static bool isExecModifyingInst(const Instruction &I) {
+ const auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB)
+ return false;
+
+ switch (CB->getIntrinsicID()) {
+ case Intrinsic::amdgcn_kill:
+ case Intrinsic::amdgcn_wqm_demote:
+ case Intrinsic::amdgcn_init_exec:
+ case Intrinsic::amdgcn_init_exec_from_input:
+ case Intrinsic::amdgcn_init_whole_wave:
+ // The WQM/WWM family makes the exec state at a given point depend on
+ // WQM/Exact decisions that SIWholeQuadMode only makes much later, so treat
+ // any occurrence of it as opaque.
+ case Intrinsic::amdgcn_wqm:
+ case Intrinsic::amdgcn_softwqm:
+ case Intrinsic::amdgcn_strict_wqm:
+ case Intrinsic::amdgcn_wwm:
+ case Intrinsic::amdgcn_strict_wwm:
+ case Intrinsic::amdgcn_set_inactive:
+ case Intrinsic::amdgcn_set_inactive_chain_arg:
+ return true;
+ case Intrinsic::not_intrinsic:
+ // A callee may itself execute llvm.amdgcn.kill.
+ return true;
+ default:
+ return false;
+ }
+}
+
+/// Returns true if exec is provably the same at \p A and at \p B, given that
+/// \p A dominates \p B. That holds when no path from \p A to \p B crosses a
+/// divergent terminator or an instruction that writes exec.
+static bool isExecInvariantBetween(const Instruction *A, const Instruction *B,
+ const UniformityInfo &UI) {
+ const BasicBlock *ABB = A->getParent();
+ const BasicBlock *BBB = B->getParent();
+
+ auto HasExecModifier = [](BasicBlock::const_iterator Begin,
+ BasicBlock::const_iterator End) {
+ return any_of(make_range(Begin, End), isExecModifyingInst);
+ };
+
+ // Straight-line case: only the instructions in between can matter. Note that
+ // if this block is part of a cycle then A re-executes before B does, so the
+ // two still pair up within an iteration.
+ if (ABB == BBB)
+ return !HasExecModifier(std::next(A->getIterator()), B->getIterator());
+
+ // Everything in BBB ahead of B runs between A and B.
+ if (HasExecModifier(BBB->begin(), B->getIterator()))
+ return false;
+
+ // Collect every block on some path ABB ->* BBB that does not re-enter ABB.
+ // Since A dominates B, every such path starts at ABB, and every block found
+ // this way is dominated by ABB. Stopping the walk at ABB is correct: if a
+ // path did revisit ABB, then a later dynamic instance of A would be the one
+ // reaching B, and the path from that instance does not revisit ABB.
+ SmallPtrSet<const BasicBlock *, 8> Region;
+ SmallVector<const BasicBlock *, 8> Worklist;
+ Region.insert(ABB);
+ for (const BasicBlock *Pred : predecessors(BBB)) {
+ if (Region.insert(Pred).second)
+ Worklist.push_back(Pred);
+ }
+
+ while (!Worklist.empty()) {
+ const BasicBlock *Cur = Worklist.pop_back_val();
+ if (Cur == ABB)
+ continue;
+ for (const BasicBlock *Pred : predecessors(Cur))
+ if (Region.insert(Pred).second)
+ Worklist.push_back(Pred);
+ }
+
+ if (Region.size() > MaxExecInvarianceBlocks)
+ return false;
+
+ // Note that BBB lands in the region only if it can reach itself without
+ // passing through ABB, i.e. B sits in a cycle that A is outside of. In that
+ // case B re-executes and its whole block, terminator included, is checked
+ // below; otherwise BBB is absent and its terminator, which runs after B, is
+ // correctly left out.
+ for (const BasicBlock *Blk : Region) {
+ if (!UI.isUniformTerminator(Blk->getTerminator()))
+ return false;
+
+ // Instructions ahead of A never run between the last A and B.
+ BasicBlock::const_iterator Begin =
+ Blk == ABB ? std::next(A->getIterator()) : Blk->begin();
+ if (HasExecModifier(Begin, Blk->end()))
+ return false;
+ }
+ return true;
+}
+
+/// Removes ballot calls that are made redundant by an earlier identical call
+/// which is guaranteed to have executed with the same exec mask.
+static bool combineRedundantBallots(Function &F, UniformityInfo &UI,
+ const DominatorTree &DT) {
+ // Bucket the ballots by (result type, condition); only calls landing in the
+ // same bucket can possibly be identical. Visiting the dominator tree in
+ // pre-order means a dominating call always precedes the calls it dominates.
+ SmallMapVector<std::pair<Type *, Value *>, SmallVector<CallInst *, 4>, 4> Buckets;
+ for (const DomTreeNode *N : depth_first(DT.getRootNode())) {
+ for (Instruction &I : *N->getBlock()) {
+ auto *CI = dyn_cast<CallInst>(&I);
+ if (CI && CI->getIntrinsicID() == Intrinsic::amdgcn_ballot)
+ Buckets[{CI->getType(), CI->getArgOperand(0)}].push_back(CI);
+ }
+ }
+
+ bool Changed = false;
+ for (auto &[Key, Ballots] : Buckets) {
+ for (unsigned I = 1, E = Ballots.size(); I != E; ++I) {
+ CallInst *B = Ballots[I];
+ for (unsigned J = 0; J < I; ++J) {
+ CallInst *A = Ballots[J];
+ // Null entries are calls that have already been erased.
+ if (!A || !A->isIdenticalToWhenDefined(B) || !DT.dominates(A, B) ||
+ !isExecInvariantBetween(A, B, UI))
+ continue;
+
+ LLVM_DEBUG(dbgs() << "Replacing redundant ballot " << *B << " with "
+ << *A << '\n');
+ UI.forgetValue(B);
+ B->replaceAllUsesWith(A);
+ B->eraseFromParent();
+ Ballots[I] = nullptr;
+ Changed = true;
+ break;
+ }
+ }
+ }
+ return Changed;
+}
+
/// Iterates over intrinsic calls in the Function to optimize.
-static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
+static bool runUniformIntrinsicCombine(Function &F, UniformityInfo &UI,
+ const DominatorTree &DT) {
bool IsChanged = false;
ValueMap<const Value *, bool> Tracker;
@@ -146,17 +301,22 @@ static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
continue;
IsChanged |= optimizeUniformIntrinsic(*II, UI, Tracker);
}
+
+ IsChanged |= combineRedundantBallots(F, UI, DT);
+
return IsChanged;
}
PreservedAnalyses
AMDGPUUniformIntrinsicCombinePass::run(Function &F,
FunctionAnalysisManager &AM) {
- const auto &UI = AM.getResult<UniformityInfoAnalysis>(F);
- if (!runUniformIntrinsicCombine(F, UI))
+ auto &UI = AM.getResult<UniformityInfoAnalysis>(F);
+ const auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+ if (!runUniformIntrinsicCombine(F, UI, DT))
return PreservedAnalyses::all();
PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
PA.preserve<UniformityInfoAnalysis>();
return PA;
}
@@ -172,6 +332,7 @@ class AMDGPUUniformIntrinsicCombineLegacy : public FunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
AU.addRequired<UniformityInfoWrapperPass>();
+ AU.addRequired<DominatorTreeWrapperPass>();
AU.addRequired<TargetPassConfig>();
}
};
@@ -184,14 +345,17 @@ char &llvm::AMDGPUUniformIntrinsicCombineLegacyPassID =
bool AMDGPUUniformIntrinsicCombineLegacy::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
- const UniformityInfo &UI =
+ UniformityInfo &UI =
getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
- return runUniformIntrinsicCombine(F, UI);
+ const DominatorTree &DT =
+ getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+ return runUniformIntrinsicCombine(F, UI, DT);
}
INITIALIZE_PASS_BEGIN(AMDGPUUniformIntrinsicCombineLegacy, DEBUG_TYPE,
"AMDGPU Uniform Intrinsic Combine", false, false)
INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_END(AMDGPUUniformIntrinsicCombineLegacy, DEBUG_TYPE,
"AMDGPU Uniform Intrinsic Combine", false, false)
diff --git a/llvm/test/CodeGen/AMDGPU/redundant-ballot-reads.ll b/llvm/test/CodeGen/AMDGPU/redundant-ballot-reads.ll
new file mode 100644
index 0000000000000..59ae73d65eb66
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/redundant-ballot-reads.ll
@@ -0,0 +1,448 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgpu11-amd-amdhsa -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s
+
+define amdgpu_kernel void @same_block(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @same_block(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[A]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @cross_block(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @cross_block(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[A]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %exit
+
+exit:
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+; A uniform branch keeps every lane together, so exec at %merge equals exec at
+; %entry.
+define amdgpu_kernel void @uniform_diamond(i1 %uc, ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_diamond(
+; CHECK-SAME: i1 [[UC:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br i1 [[UC]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: br label %[[MERGE:.*]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[A]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br i1 %uc, label %then, label %else
+
+then:
+ br label %merge
+
+else:
+ br label %merge
+
+merge:
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+; The loop is uniform, so every lane iterates in lockstep and the ballot in the
+; body always sees the exec mask it had on entry.
+define amdgpu_kernel void @uniform_loop(i32 %n, ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_loop(
+; CHECK-SAME: i32 [[N:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[A]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: [[NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %next, %loop ]
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ %next = add i32 %iv, 1
+ %ec = icmp eq i32 %next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; All three collapse onto the first.
+define amdgpu_kernel void @chain(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @chain(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[MID:.*]]
+; CHECK: [[MID]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R0:%.*]] = add i64 [[A]], [[A]]
+; CHECK-NEXT: [[R1:%.*]] = add i64 [[R0]], [[A]]
+; CHECK-NEXT: store i64 [[R1]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %mid
+
+mid:
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %exit
+
+exit:
+ %c = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r0 = add i64 %a, %b
+ %r1 = add i64 %r0, %c
+ store i64 %r1, ptr addrspace(1) %out
+ ret void
+}
+
+; Negative test: lanes reconverge at %merge, but a divergent branch is
+; not something this pass reasons about, so both calls stay.
+define amdgpu_kernel void @divergent_branch(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @divergent_branch(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: [[DC:%.*]] = icmp ult i32 [[TID]], 16
+; CHECK-NEXT: br i1 [[DC]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: br label %[[MERGE:.*]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[B]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %dc = icmp ult i32 %tid, 16
+ br i1 %dc, label %then, label %else
+
+then:
+ br label %merge
+
+else:
+ br label %merge
+
+merge:
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+; Negative test: The divergent latch does not dominate %loop, so an
+; idom-chain walk would miss it. Lanes drop out of the loop one at
+; a time, so the ballot in %loop sees a different exec mask on later
+; iterations.
+define amdgpu_kernel void @divergent_latch(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @divergent_latch(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[B]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ult i32 [[NEXT]], [[TID]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %next, %latch ]
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ br label %latch
+
+latch:
+ %next = add i32 %iv, 1
+ %ec = icmp ult i32 %next, %tid
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define amdgpu_kernel void @different_arg(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @different_arg(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND0:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[COND1:%.*]] = icmp ult i32 [[TID]], 16
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND0]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND1]])
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[B]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond0 = icmp ult i32 %tid, 8
+ %cond1 = icmp ult i32 %tid, 16
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond0)
+ br label %exit
+
+exit:
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond1)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @different_type(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @different_type(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[B:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[COND]])
+; CHECK-NEXT: [[BZ:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[BZ]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %exit
+
+exit:
+ %b = call i32 @llvm.amdgcn.ballot.i32(i1 %cond)
+ %bz = zext i32 %b to i64
+ %r = add i64 %a, %bz
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kill_between(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @kill_between(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: call void @llvm.amdgcn.kill(i1 [[COND]])
+; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[B]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %exit
+
+exit:
+ call void @llvm.amdgcn.kill(i1 %cond)
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+; The callee could itself execute llvm.amdgcn.kill.
+define amdgpu_kernel void @opaque_call_between(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @opaque_call_between(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: call void @external()
+; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[B]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %exit
+
+exit:
+ call void @external()
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
+; Neither call dominates the other.
+define amdgpu_kernel void @no_dominance(i1 %uc, ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @no_dominance(
+; CHECK-SAME: i1 [[UC:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: br i1 [[UC]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[MERGE:.*]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]])
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[P:%.*]] = phi i64 [ [[A]], %[[THEN]] ], [ [[B]], %[[ELSE]] ]
+; CHECK-NEXT: store i64 [[P]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ br i1 %uc, label %then, label %else
+
+then:
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %merge
+
+else:
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond)
+ br label %merge
+
+merge:
+ %p = phi i64 [ %a, %then ], [ %b, %else ]
+ store i64 %p, ptr addrspace(1) %out
+ ret void
+}
+
+; The two calls sit in different convergence regions, so they are not identical
+; and must not be merged even though the loop itself is uniform.
+define amdgpu_kernel void @convergence_tokens(i32 %n, ptr addrspace(1) %out) convergent {
+; CHECK-LABEL: define amdgpu_kernel void @convergence_tokens(
+; CHECK-SAME: i32 [[N:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[T:%.*]] = call token @llvm.experimental.convergence.entry()
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[TID]], 8
+; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]]) [ "convergencectrl"(token [[T]]) ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LT:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T]]) ]
+; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[COND]]) [ "convergencectrl"(token [[LT]]) ]
+; CHECK-NEXT: [[R:%.*]] = add i64 [[A]], [[B]]
+; CHECK-NEXT: store i64 [[R]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: [[NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %t = call token @llvm.experimental.convergence.entry()
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %cond = icmp ult i32 %tid, 8
+ %a = call i64 @llvm.amdgcn.ballot.i64(i1 %cond) [ "convergencectrl"(token %t) ]
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %next, %loop ]
+ %lt = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t) ]
+ %b = call i64 @llvm.amdgcn.ballot.i64(i1 %cond) [ "convergencectrl"(token %lt) ]
+ %r = add i64 %a, %b
+ store i64 %r, ptr addrspace(1) %out
+ %next = add i32 %iv, 1
+ %ec = icmp eq i32 %next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare void @external()
More information about the llvm-commits
mailing list