[llvm] [SPIRV] Add handling for `uinc_wrap` and `udec_wrap` atomics (PR #179114)
Lleu Yang via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 4 03:38:59 PST 2026
https://github.com/megakite updated https://github.com/llvm/llvm-project/pull/179114
>From 38fb452da500c37cf13fc8c73f8dd6a0fb6e04f7 Mon Sep 17 00:00:00 2001
From: Lleu Yang <hello at megakite.icu>
Date: Sun, 1 Feb 2026 23:00:06 +0800
Subject: [PATCH 1/4] [SPIRV] Add handling for `uinc_wrap` and `udec_wrap`
atomics
This adds atomicrmw `uinc_wrap` and `udec_wrap` operations support for
SPIR-V. Since SPIR-V doesn't provide dedicated instructions for those
two operations, we have to use the `AtomicExpand` pass to expand the
operations into CAS forms.
Closes #177204.
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 32 ++++++++-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 5 ++
llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 2 +
.../CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 68 +++++++++++++++++++
4 files changed, 106 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 3e5ce4b90ea4a..6455eadc7bc72 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -18,6 +18,8 @@
#include "SPIRVSubtarget.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#define DEBUG_TYPE "spirv-lower"
@@ -26,7 +28,12 @@ using namespace llvm;
SPIRVTargetLowering::SPIRVTargetLowering(const TargetMachine &TM,
const SPIRVSubtarget &ST)
- : TargetLowering(TM, ST), STI(ST) {}
+ : TargetLowering(TM, ST), STI(ST) {
+ // Avoid AtomicExpand treating all atomics as unsupported and lowering them
+ // to libcalls.
+ setMaxAtomicSizeInBitsSupported(64);
+ setMinCmpXchgSizeInBits(32);
+}
// Returns true of the types logically match, as defined in
// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpCopyLogical.
@@ -626,3 +633,26 @@ bool SPIRVTargetLowering::insertLogicalCopyOnResult(
.constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(),
*STI.getRegBankInfo());
}
+
+TargetLowering::AtomicExpansionKind
+SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+ switch (RMW->getOperation()) {
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ case AtomicRMWInst::FMin:
+ case AtomicRMWInst::FMax:
+ return AtomicExpansionKind::None;
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
+ return AtomicExpansionKind::CmpXChg;
+ default:
+ return TargetLowering::shouldExpandAtomicRMWInIR(RMW);
+ }
+}
+
+TargetLowering::AtomicExpansionKind
+SPIRVTargetLowering::shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
+ // Do not cast atomic exchange at all since SPIR-V natively supports
+ // floating-point and pointer exchanges.
+ return AtomicExpansionKind::None;
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index 462605ab6fe36..7390afc1223ea 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -75,6 +75,11 @@ class SPIRVTargetLowering : public TargetLowering {
unsigned OpIdx) const;
bool insertLogicalCopyOnResult(MachineInstr &I,
SPIRVType *NewResultType) const;
+
+ AtomicExpansionKind
+ shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override;
+ AtomicExpansionKind
+ shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const override;
};
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index 2d70972d6fbdb..301fe3d487565 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -174,6 +174,8 @@ TargetPassConfig *SPIRVTargetMachine::createPassConfig(PassManagerBase &PM) {
}
void SPIRVPassConfig::addIRPasses() {
+ addPass(createAtomicExpandLegacyPass());
+
TargetPassConfig::addIRPasses();
addPass(createSPIRVRegularizerPass());
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
new file mode 100644
index 0000000000000..3754470eefae7
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -0,0 +1,68 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Bool:]] = OpTypeBool
+; CHECK-DAG: %[[#Struct:]] = OpTypeStruct %[[#Int]] %[[#Bool]]
+; CHECK-DAG: %[[#StructPtr:]] = OpTypePointer CrossWorkgroup %[[#Struct]]
+; CHECK-DAG: %[[#IntPtr:]] = OpTypePointer CrossWorkgroup %[[#Int]]
+; CHECK-DAG: %[[#UndefStruct:]] = OpUndef %[[#Struct]]
+; CHECK-DAG: %[[#MemSem:]] = OpConstant %[[#Int]] 16
+; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
+; CHECK-DAG: %[[#One:]] = OpConstant %[[#Int]] 1
+; CHECK-DAG: %[[#Scope:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#Pointer:]] = OpVariable %[[#IntPtr]] CrossWorkgroup
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @atomicrmw_uinc_udec_wrap() local_unnamed_addr {
+entry:
+; CHECK: %[[#Entry:]] = OpLabel
+; CHECK: %[[#Cast1:]] = OpBitcast %[[#StructPtr]] %[[#Pointer]]
+; CHECK: %[[#Cast2:]] = OpBitcast %[[#StructPtr]] %[[#Pointer]]
+
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+; CHECK: %[[#LoadUi1:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
+; CHECK: OpBranch %[[#Loop1:]]
+; CHECK: %[[#Loop1]] = OpLabel
+; CHECK: %[[#Phi1:]] = OpPhi %[[#Int]] %[[#LoadUi1]] %[[#Entry]] %[[#Phi1Next:]] %[[#Loop1]]
+; CHECK: %[[#Add:]] = OpIAdd %[[#Int]] %[[#Phi1]] %[[#One]]
+; CHECK: %[[#GE:]] = OpUGreaterThanEqual %[[#Bool]] %[[#Phi1]] %[[#Value]]
+; CHECK: %[[#Select1:]] = OpSelect %[[#Int]] %[[#GE]] %[[#Scope]] %[[#Add]]
+; CHECK: %[[#Bitcast1:]] = OpBitcast %[[#IntPtr]] %[[#Cast1]]
+; CHECK: %[[#CmpXChg1:]] = OpAtomicCompareExchange %[[#Int]] %[[#Bitcast1]] %[[#Scope]] %[[#MemSem]] %[[#MemSem]] %[[#Select1]] %[[#Phi1]]
+; CHECK: %[[#Eq1:]] = OpIEqual %[[#Bool]] %[[#CmpXChg1]] %[[#Phi1]]
+; CHECK: %[[#Insert1:]] = OpCompositeInsert %[[#Struct]] %[[#CmpXChg1]] %[[#UndefStruct]] 0
+; CHECK: %[[#Insert2:]] = OpCompositeInsert %[[#Struct]] %[[#Eq1]] %[[#Insert1]] 1
+; CHECK: %[[#Cond1:]] = OpCompositeExtract %[[#Bool]] %[[#Insert2]] 1
+; CHECK: %[[#Phi1Next]] = OpCompositeExtract %[[#Int]] %[[#Insert2]] 0
+; CHECK: OpBranchConditional %[[#Cond1]] %[[#Exit1:]] %[[#Loop1]]
+; CHECK: %[[#Exit1]] = OpLabel
+
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+; CHECK: %[[#LoadUi2:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
+; CHECK: OpBranch %[[#Loop2:]]
+; CHECK: %[[#Loop2]] = OpLabel
+; CHECK: %[[#Phi2:]] = OpPhi %[[#Int]] %[[#LoadUi2]] %[[#Exit1]] %[[#Phi2Next:]] %[[#Loop2]]
+; CHECK: %[[#Sub:]] = OpISub %[[#Int]] %[[#Phi2]] %[[#One]]
+; CHECK: %[[#Eq2:]] = OpIEqual %[[#Bool]] %[[#Phi2]] %[[#Scope]]
+; CHECK: %[[#GT:]] = OpUGreaterThan %[[#Bool]] %[[#Phi2]] %[[#Value]]
+; CHECK: %[[#Or:]] = OpLogicalOr %[[#Bool]] %[[#Eq2]] %[[#GT]]
+; CHECK: %[[#Select2:]] = OpSelect %[[#Int]] %[[#Or]] %[[#Value]] %[[#Sub]]
+; CHECK: %[[#Bitcast2:]] = OpBitcast %[[#IntPtr]] %[[#Cast2]]
+; CHECK: %[[#CmpXChg2:]] = OpAtomicCompareExchange %[[#Int]] %[[#Bitcast2]] %[[#Scope]] %[[#MemSem]] %[[#MemSem]] %[[#Select2]] %[[#Phi2]]
+; CHECK: %[[#Eq3:]] = OpIEqual %[[#Bool]] %[[#CmpXChg2]] %[[#Phi2]]
+; CHECK: %[[#Insert3:]] = OpCompositeInsert %[[#Struct]] %[[#CmpXChg2]] %[[#UndefStruct]] 0
+; CHECK: %[[#Insert4:]] = OpCompositeInsert %[[#Struct]] %[[#Eq3]] %[[#Insert3]] 1
+; CHECK: %[[#Cond2:]] = OpCompositeExtract %[[#Bool]] %[[#Insert4]] 1
+; CHECK: %[[#Phi2Next]] = OpCompositeExtract %[[#Int]] %[[#Insert4]] 0
+; CHECK: OpBranchConditional %[[#Cond2]] %[[#Exit2:]] %[[#Loop2]]
+; CHECK: %[[#Exit2]] = OpLabel
+
+ ret void
+; CHECK: OpReturn
+; CHECK: OpFunctionEnd
+}
>From 6ef18fbfe3aacbfcfb6d4516d97d1130b3afc05f Mon Sep 17 00:00:00 2001
From: Lleu Yang <hello at megakite.icu>
Date: Mon, 2 Feb 2026 09:55:44 +0800
Subject: [PATCH 2/4] [SPIRV] Update llc pipeline test
Added "Expand Atomic instructions" into checks.
---
llvm/test/CodeGen/SPIRV/llc-pipeline.ll | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
index 2f8b9decaaba3..70ae7081ad5d5 100644
--- a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
+++ b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
@@ -24,6 +24,7 @@
; SPIRV-O0-NEXT: Pre-ISel Intrinsic Lowering
; SPIRV-O0-NEXT: FunctionPass Manager
; SPIRV-O0-NEXT: Expand IR instructions
+; SPIRV-O0-NEXT: Expand Atomic instructions
; SPIRV-O0-NEXT: Lower Garbage Collection Instructions
; SPIRV-O0-NEXT: Shadow Stack GC Lowering
; SPIRV-O0-NEXT: Remove unreachable blocks from the CFG
@@ -102,6 +103,7 @@
; SPIRV-Opt-NEXT: Pre-ISel Intrinsic Lowering
; SPIRV-Opt-NEXT: FunctionPass Manager
; SPIRV-Opt-NEXT: Expand IR instructions
+; SPIRV-Opt-NEXT: Expand Atomic instructions
; SPIRV-Opt-NEXT: Dominator Tree Construction
; SPIRV-Opt-NEXT: Basic Alias Analysis (stateless AA impl)
; SPIRV-Opt-NEXT: Natural Loop Information
>From fbdafd09d6db3db7c16c9114bb0fa3bd728a50f4 Mon Sep 17 00:00:00 2001
From: Lleu Yang <hello at megakite.icu>
Date: Wed, 4 Feb 2026 18:10:55 +0800
Subject: [PATCH 3/4] [SPIRV] Change min CmpXchg size to 8 and add
corresponding tests
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 8 +++++---
.../CodeGen/SPIRV/AtomicCompareExchange.ll | 19 +++++++++++++++++++
2 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 6455eadc7bc72..b8c0ba3564725 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -29,10 +29,12 @@ using namespace llvm;
SPIRVTargetLowering::SPIRVTargetLowering(const TargetMachine &TM,
const SPIRVSubtarget &ST)
: TargetLowering(TM, ST), STI(ST) {
- // Avoid AtomicExpand treating all atomics as unsupported and lowering them
- // to libcalls.
+ // Even with SPV_ALTERA_arbitrary_precision_integers enabled, atomic sizes are
+ // limited by atomicrmw xchg operation, which only supports operand up to 64
+ // bits wide, as defined in SPIR-V legalizer. Currently, spirv-val doesn't
+ // consider 128-bit OpTypeInt as valid either.
setMaxAtomicSizeInBitsSupported(64);
- setMinCmpXchgSizeInBits(32);
+ setMinCmpXchgSizeInBits(8);
}
// Returns true of the types logically match, as defined in
diff --git a/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll b/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll
index b92c90c993c7c..da384f3f4f250 100644
--- a/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll
+++ b/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll
@@ -10,6 +10,11 @@
; CHECK-SPIRV-DAG: %[[#Bool:]] = OpTypeBool
; CHECK-SPIRV-DAG: %[[#Struct:]] = OpTypeStruct %[[#Int]] %[[#Bool]]
; CHECK-SPIRV-DAG: %[[#UndefStruct:]] = OpUndef %[[#Struct]]
+; CHECK-SPIRV-DAG: %[[#Int8:]] = OpTypeInt 8 0
+; CHECK-SPIRV-DAG: %[[#Constant_45:]] = OpConstant %[[#Int8]] 45{{$}}
+; CHECK-SPIRV-DAG: %[[#Constant_12:]] = OpConstant %[[#Int8]] 12{{$}}
+; CHECK-SPIRV-DAG: %[[#Struct8:]] = OpTypeStruct %[[#Int8]] %[[#Bool]]
+; CHECK-SPIRV-DAG: %[[#UndefStruct8:]] = OpUndef %[[#Struct8]]
; CHECK-SPIRV: %[[#Value:]] = OpLoad %[[#Int]] %[[#Value_ptr:]]
; CHECK-SPIRV: %[[#Res:]] = OpAtomicCompareExchange %[[#Int]] %[[#Pointer:]] %[[#MemScope_CrossDevice]]
@@ -48,3 +53,17 @@ entry:
store { i32, i1 } %0, ptr %store_ptr, align 4
ret void
}
+
+; CHECK-SPIRV: %[[#Res_2:]] = OpAtomicCompareExchange %[[#Int8]] %[[#Ptr:]] %[[#MemScope_CrossDevice]]
+; CHECK-SPIRV-SAME: %[[#MemSemEqual_SeqCst]] %[[#MemSemUnequal_Acquire]] %[[#Constant_45]] %[[#Constant_12]]
+; CHECK-SPIRV: %[[#Success_2:]] = OpIEqual %[[#]] %[[#Res_2]] %[[#Constant_12]]
+; CHECK-SPIRV: %[[#Composite_2:]] = OpCompositeInsert %[[#Struct8]] %[[#Res_2]] %[[#UndefStruct8]] 0
+; CHECK-SPIRV: %[[#Composite_3:]] = OpCompositeInsert %[[#Struct8]] %[[#Success_2]] %[[#Composite_2]] 1
+; CHECK-SPIRV: OpStore %[[#Store_ptr:]] %[[#Composite_3]]
+
+define dso_local spir_func void @test3(ptr %ptr, ptr %store_ptr) local_unnamed_addr {
+entry:
+ %0 = cmpxchg ptr %ptr, i8 12, i8 45 seq_cst acquire
+ store { i8, i1 } %0, ptr %store_ptr, align 1
+ ret void
+}
>From 6f41cf51e459c8d3e947faefa391342aa289c113 Mon Sep 17 00:00:00 2001
From: Lleu Yang <hello at megakite.icu>
Date: Wed, 4 Feb 2026 19:28:54 +0800
Subject: [PATCH 4/4] [SPIRV] Add TODO comment in shouldCastAtomicRMWInIR()
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index b8c0ba3564725..7d15627b517ba 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -654,7 +654,7 @@ SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
TargetLowering::AtomicExpansionKind
SPIRVTargetLowering::shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
- // Do not cast atomic exchange at all since SPIR-V natively supports
- // floating-point and pointer exchanges.
+ // TODO: Pointer operand should be cast to integer in atomicrmw xchg, since
+ // SPIR-V only supports atomic exchange for integer and floating-point types.
return AtomicExpansionKind::None;
}
More information about the llvm-commits
mailing list