[llvm] [SPIRV] Add handling for `uinc_wrap` and `udec_wrap` atomics (PR #179114)
Lleu Yang via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 1 07:22:08 PST 2026
https://github.com/megakite created https://github.com/llvm/llvm-project/pull/179114
This adds atomicrmw `uinc_wrap` and `udec_wrap` operations support for SPIR-V. Since SPIR-V doesn't provide dedicated instructions for those two operations, we have to use the `AtomicExpand` pass to expand the operations into CAS forms.
Closes #177204.
>From e06c0473527fc4fdeca2846f848f09a7809fd7cc Mon Sep 17 00:00:00 2001
From: Lleu Yang <hello at megakite.icu>
Date: Sun, 1 Feb 2026 23:00:06 +0800
Subject: [PATCH] [SPIRV] Add handling for `uinc_wrap` and `udec_wrap` atomics
This adds atomicrmw `uinc_wrap` and `udec_wrap` operations support for
SPIR-V. Since SPIR-V doesn't provide dedicated instructions for those
two operations, we have to use the `AtomicExpand` pass to expand the
operations into CAS forms.
Closes #177204.
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 32 ++++++++-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 5 ++
llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 2 +
.../CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 68 +++++++++++++++++++
4 files changed, 106 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 36fa5fa9a70cb..d7762f075363a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -18,6 +18,8 @@
#include "SPIRVSubtarget.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#define DEBUG_TYPE "spirv-lower"
@@ -26,7 +28,12 @@ using namespace llvm;
SPIRVTargetLowering::SPIRVTargetLowering(const TargetMachine &TM,
const SPIRVSubtarget &ST)
- : TargetLowering(TM, ST), STI(ST) {}
+ : TargetLowering(TM, ST), STI(ST) {
+ // Avoid AtomicExpand treating all atomics as unsupported and lowering them
+ // to libcalls.
+ setMaxAtomicSizeInBitsSupported(64);
+ setMinCmpXchgSizeInBits(32);
+}
// Returns true of the types logically match, as defined in
// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpCopyLogical.
@@ -627,3 +634,26 @@ bool SPIRVTargetLowering::insertLogicalCopyOnResult(
.constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(),
*STI.getRegBankInfo());
}
+
+TargetLowering::AtomicExpansionKind
+SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+ switch (RMW->getOperation()) {
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ case AtomicRMWInst::FMin:
+ case AtomicRMWInst::FMax:
+ return AtomicExpansionKind::None;
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
+ return AtomicExpansionKind::CmpXChg;
+ default:
+ return TargetLowering::shouldExpandAtomicRMWInIR(RMW);
+ }
+}
+
+TargetLowering::AtomicExpansionKind
+SPIRVTargetLowering::shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
+ // Do not cast atomic exchange at all since SPIR-V natively supports
+ // floating-point and pointer exchanges.
+ return AtomicExpansionKind::None;
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index 5746832c8fd95..df06ddcb88607 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -75,6 +75,11 @@ class SPIRVTargetLowering : public TargetLowering {
unsigned OpIdx) const;
bool insertLogicalCopyOnResult(MachineInstr &I,
SPIRVType *NewResultType) const;
+
+ AtomicExpansionKind
+ shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override;
+ AtomicExpansionKind
+ shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const override;
};
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index 2d70972d6fbdb..301fe3d487565 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -174,6 +174,8 @@ TargetPassConfig *SPIRVTargetMachine::createPassConfig(PassManagerBase &PM) {
}
void SPIRVPassConfig::addIRPasses() {
+ addPass(createAtomicExpandLegacyPass());
+
TargetPassConfig::addIRPasses();
addPass(createSPIRVRegularizerPass());
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
new file mode 100644
index 0000000000000..3754470eefae7
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -0,0 +1,68 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Bool:]] = OpTypeBool
+; CHECK-DAG: %[[#Struct:]] = OpTypeStruct %[[#Int]] %[[#Bool]]
+; CHECK-DAG: %[[#StructPtr:]] = OpTypePointer CrossWorkgroup %[[#Struct]]
+; CHECK-DAG: %[[#IntPtr:]] = OpTypePointer CrossWorkgroup %[[#Int]]
+; CHECK-DAG: %[[#UndefStruct:]] = OpUndef %[[#Struct]]
+; CHECK-DAG: %[[#MemSem:]] = OpConstant %[[#Int]] 16
+; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
+; CHECK-DAG: %[[#One:]] = OpConstant %[[#Int]] 1
+; CHECK-DAG: %[[#Scope:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#Pointer:]] = OpVariable %[[#IntPtr]] CrossWorkgroup
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @atomicrmw_uinc_udec_wrap() local_unnamed_addr {
+entry:
+; CHECK: %[[#Entry:]] = OpLabel
+; CHECK: %[[#Cast1:]] = OpBitcast %[[#StructPtr]] %[[#Pointer]]
+; CHECK: %[[#Cast2:]] = OpBitcast %[[#StructPtr]] %[[#Pointer]]
+
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+; CHECK: %[[#LoadUi1:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
+; CHECK: OpBranch %[[#Loop1:]]
+; CHECK: %[[#Loop1]] = OpLabel
+; CHECK: %[[#Phi1:]] = OpPhi %[[#Int]] %[[#LoadUi1]] %[[#Entry]] %[[#Phi1Next:]] %[[#Loop1]]
+; CHECK: %[[#Add:]] = OpIAdd %[[#Int]] %[[#Phi1]] %[[#One]]
+; CHECK: %[[#GE:]] = OpUGreaterThanEqual %[[#Bool]] %[[#Phi1]] %[[#Value]]
+; CHECK: %[[#Select1:]] = OpSelect %[[#Int]] %[[#GE]] %[[#Scope]] %[[#Add]]
+; CHECK: %[[#Bitcast1:]] = OpBitcast %[[#IntPtr]] %[[#Cast1]]
+; CHECK: %[[#CmpXChg1:]] = OpAtomicCompareExchange %[[#Int]] %[[#Bitcast1]] %[[#Scope]] %[[#MemSem]] %[[#MemSem]] %[[#Select1]] %[[#Phi1]]
+; CHECK: %[[#Eq1:]] = OpIEqual %[[#Bool]] %[[#CmpXChg1]] %[[#Phi1]]
+; CHECK: %[[#Insert1:]] = OpCompositeInsert %[[#Struct]] %[[#CmpXChg1]] %[[#UndefStruct]] 0
+; CHECK: %[[#Insert2:]] = OpCompositeInsert %[[#Struct]] %[[#Eq1]] %[[#Insert1]] 1
+; CHECK: %[[#Cond1:]] = OpCompositeExtract %[[#Bool]] %[[#Insert2]] 1
+; CHECK: %[[#Phi1Next]] = OpCompositeExtract %[[#Int]] %[[#Insert2]] 0
+; CHECK: OpBranchConditional %[[#Cond1]] %[[#Exit1:]] %[[#Loop1]]
+; CHECK: %[[#Exit1]] = OpLabel
+
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+; CHECK: %[[#LoadUi2:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
+; CHECK: OpBranch %[[#Loop2:]]
+; CHECK: %[[#Loop2]] = OpLabel
+; CHECK: %[[#Phi2:]] = OpPhi %[[#Int]] %[[#LoadUi2]] %[[#Exit1]] %[[#Phi2Next:]] %[[#Loop2]]
+; CHECK: %[[#Sub:]] = OpISub %[[#Int]] %[[#Phi2]] %[[#One]]
+; CHECK: %[[#Eq2:]] = OpIEqual %[[#Bool]] %[[#Phi2]] %[[#Scope]]
+; CHECK: %[[#GT:]] = OpUGreaterThan %[[#Bool]] %[[#Phi2]] %[[#Value]]
+; CHECK: %[[#Or:]] = OpLogicalOr %[[#Bool]] %[[#Eq2]] %[[#GT]]
+; CHECK: %[[#Select2:]] = OpSelect %[[#Int]] %[[#Or]] %[[#Value]] %[[#Sub]]
+; CHECK: %[[#Bitcast2:]] = OpBitcast %[[#IntPtr]] %[[#Cast2]]
+; CHECK: %[[#CmpXChg2:]] = OpAtomicCompareExchange %[[#Int]] %[[#Bitcast2]] %[[#Scope]] %[[#MemSem]] %[[#MemSem]] %[[#Select2]] %[[#Phi2]]
+; CHECK: %[[#Eq3:]] = OpIEqual %[[#Bool]] %[[#CmpXChg2]] %[[#Phi2]]
+; CHECK: %[[#Insert3:]] = OpCompositeInsert %[[#Struct]] %[[#CmpXChg2]] %[[#UndefStruct]] 0
+; CHECK: %[[#Insert4:]] = OpCompositeInsert %[[#Struct]] %[[#Eq3]] %[[#Insert3]] 1
+; CHECK: %[[#Cond2:]] = OpCompositeExtract %[[#Bool]] %[[#Insert4]] 1
+; CHECK: %[[#Phi2Next]] = OpCompositeExtract %[[#Int]] %[[#Insert4]] 0
+; CHECK: OpBranchConditional %[[#Cond2]] %[[#Exit2:]] %[[#Loop2]]
+; CHECK: %[[#Exit2]] = OpLabel
+
+ ret void
+; CHECK: OpReturn
+; CHECK: OpFunctionEnd
+}
More information about the llvm-commits
mailing list