[llvm] [AMDGPU] Skip functions with convergence tokens in AtomicOptimizer (PR #227929)
Mian Miftah via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 18:47:23 PDT 2026
https://github.com/mmiftahx created https://github.com/llvm/llvm-project/pull/227929
AMDGPUAtomicOptimizer inserts convergent calls such as ballot, readlane
and wave.reduce without a convergencectrl bundle. In a function that
uses convergence control tokens, the verifier then rejects the result:
"Cannot mix controlled and uncontrolled convergence in the same
function". llc's default pipeline hits this, for example with a
controlled readfirstlane and a uniform atomicrmw add in one kernel.
Leave such functions alone, as the inliner does when it can't supply a
token (the "convergent call needs convergencectrl operand" check in
InlineFunction.cpp).
>From 83bb43f72a9496b86e851144d21e2c3c580bf490 Mon Sep 17 00:00:00 2001
From: mmiftahx <mmiftah.duna at gmail.com>
Date: Wed, 30 Sep 2026 11:46:47 -0500
Subject: [PATCH] [AMDGPU] Skip functions with convergence tokens in
AtomicOptimizer
AMDGPUAtomicOptimizer inserts convergent calls such as ballot, readlane
and wave.reduce without a convergencectrl bundle. In a function that
uses convergence control tokens, the verifier then rejects the result:
"Cannot mix controlled and uncontrolled convergence in the same
function". llc's default pipeline hits this, for example with a
controlled readfirstlane and a uniform atomicrmw add in one kernel.
Leave such functions alone, as the inliner does when it can't supply a
token (the "convergent call needs convergencectrl operand" check in
InlineFunction.cpp).
---
.../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 6 ++
.../atomic-optimizer-convergence-tokens.ll | 80 +++++++++++++++++++
2 files changed, 86 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/atomic-optimizer-convergence-tokens.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index aa822c13dd8fe..9b95abd9c24d3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -27,6 +27,7 @@
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/InitializePasses.h"
@@ -162,6 +163,11 @@ bool AMDGPUAtomicOptimizerImpl::run() {
if (ToReplace.empty())
return false;
+ // The new cross-lane operations are convergent and carry no convergence
+ // control token, so they must not be added to a function that uses tokens.
+ if (any_of(instructions(F), IsaPred<ConvergenceControlInst>))
+ return false;
+
for (auto &[I, Op, ValIdx, ValDivergent, IsLDS] : ToReplace)
optimizeAtomic(*I, Op, ValIdx, ValDivergent, IsLDS);
ToReplace.clear();
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-convergence-tokens.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-convergence-tokens.ll
new file mode 100644
index 0000000000000..096ca2cf8f815
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-convergence-tokens.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=iterative>,verify<domtree>' %s | FileCheck --check-prefixes=IR,IR-ITER %s
+; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=dpp>,verify<domtree>' %s | FileCheck --check-prefixes=IR,IR-DPP %s
+
+; The optimizer adds convergent operations without convergence control
+; tokens, which can't be mixed with controlled ones, so functions that use
+; tokens are left alone.
+
+define amdgpu_kernel void @controlled(ptr addrspace(1) %p, ptr addrspace(1) %q) convergent {
+; IR-LABEL: define amdgpu_kernel void @controlled(
+; IR-SAME: ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-NEXT: [[ENTRY:.*:]]
+; IR-NEXT: [[T:%.*]] = call token @llvm.experimental.convergence.entry()
+; IR-NEXT: [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; IR-NEXT: [[U:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ID]]) [ "convergencectrl"(token [[T]]) ]
+; IR-NEXT: store i32 [[U]], ptr addrspace(1) [[Q]], align 4
+; IR-NEXT: [[OLD:%.*]] = atomicrmw add ptr addrspace(1) [[P]], i32 1 syncscope("agent") monotonic, align 4
+; IR-NEXT: ret void
+;
+entry:
+ %t = call token @llvm.experimental.convergence.entry()
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ %u = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %id) [ "convergencectrl"(token %t) ]
+ store i32 %u, ptr addrspace(1) %q, align 4
+ %old = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, align 4
+ ret void
+}
+
+; Without tokens, the same code is still optimized.
+
+define amdgpu_kernel void @uncontrolled(ptr addrspace(1) %p, ptr addrspace(1) %q) {
+; IR-ITER-LABEL: define amdgpu_kernel void @uncontrolled(
+; IR-ITER-SAME: ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[Q:%.*]]) {
+; IR-ITER-NEXT: [[ENTRY:.*:]]
+; IR-ITER-NEXT: [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; IR-ITER-NEXT: [[U:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ID]])
+; IR-ITER-NEXT: store i32 [[U]], ptr addrspace(1) [[Q]], align 4
+; IR-ITER-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITER-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-ITER-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-ITER-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-ITER-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-ITER-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-ITER-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
+; IR-ITER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-ITER-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR-ITER: [[BB8]]:
+; IR-ITER-NEXT: [[TMP9:%.*]] = atomicrmw add ptr addrspace(1) [[P]], i32 [[TMP6]] syncscope("agent") monotonic, align 4
+; IR-ITER-NEXT: br label %[[BB10]]
+; IR-ITER: [[BB10]]:
+; IR-ITER-NEXT: ret void
+;
+; IR-DPP-LABEL: define amdgpu_kernel void @uncontrolled(
+; IR-DPP-SAME: ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[Q:%.*]]) {
+; IR-DPP-NEXT: [[ENTRY:.*:]]
+; IR-DPP-NEXT: [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; IR-DPP-NEXT: [[U:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ID]])
+; IR-DPP-NEXT: store i32 [[U]], ptr addrspace(1) [[Q]], align 4
+; IR-DPP-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-DPP-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-DPP-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-DPP-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 2)
+; IR-DPP-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-DPP-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR-DPP: [[BB8]]:
+; IR-DPP-NEXT: [[TMP9:%.*]] = atomicrmw add ptr addrspace(1) [[P]], i32 [[TMP6]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label %[[BB10]]
+; IR-DPP: [[BB10]]:
+; IR-DPP-NEXT: ret void
+;
+entry:
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ %u = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %id)
+ store i32 %u, ptr addrspace(1) %q, align 4
+ %old = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, align 4
+ ret void
+}
More information about the llvm-commits
mailing list