[llvm] 2a93d3d - [NVPTX] Support native 64-bit atomic add/sub pre-SM32 (#222471)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 15:48:58 PDT 2026
Author: Yonah Goldberg
Date: 2026-09-30T15:48:51-07:00
New Revision: 2a93d3d75630341539a5495ea07b6a0782eaebf7
URL: https://github.com/llvm/llvm-project/commit/2a93d3d75630341539a5495ea07b6a0782eaebf7
DIFF: https://github.com/llvm/llvm-project/commit/2a93d3d75630341539a5495ea07b6a0782eaebf7.diff
LOG: [NVPTX] Support native 64-bit atomic add/sub pre-SM32 (#222471)
64-bit min/max/and/or/xor require SM32 but add/sub don't.
Added:
llvm/test/Transforms/AtomicExpand/NVPTX/atomicrmw-i64-sm20.ll
Modified:
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
llvm/lib/Target/NVPTX/NVPTXSubtarget.h
Removed:
################################################################################
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 7ac0ac501000c..cde47c7ab3d9a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7777,26 +7777,23 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
if (BitWidth == 128)
return AtomicExpansionKind::None;
[[fallthrough]];
- case AtomicRMWInst::BinOp::And:
- case AtomicRMWInst::BinOp::Or:
- case AtomicRMWInst::BinOp::Xor:
+ case AtomicRMWInst::BinOp::Add:
+ case AtomicRMWInst::BinOp::Sub:
switch (BitWidth) {
case 8:
case 16:
return AtomicExpansionKind::CmpXChg;
case 32:
- return AtomicExpansionKind::None;
case 64:
- if (STI.hasAtomBitwise64())
- return AtomicExpansionKind::None;
- return AtomicExpansionKind::CmpXChg;
+ return AtomicExpansionKind::None;
case 128:
return AtomicExpansionKind::CmpXChg;
default:
llvm_unreachable("unsupported width encountered");
}
- case AtomicRMWInst::BinOp::Add:
- case AtomicRMWInst::BinOp::Sub:
+ case AtomicRMWInst::BinOp::And:
+ case AtomicRMWInst::BinOp::Or:
+ case AtomicRMWInst::BinOp::Xor:
case AtomicRMWInst::BinOp::Max:
case AtomicRMWInst::BinOp::Min:
case AtomicRMWInst::BinOp::UMax:
@@ -7808,7 +7805,7 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
case 32:
return AtomicExpansionKind::None;
case 64:
- if (STI.hasAtomMinMax64())
+ if (STI.hasAtomMinMaxAndOrXor())
return AtomicExpansionKind::None;
return AtomicExpansionKind::CmpXChg;
case 128:
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 8973b3b9fd662..953572e719305 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -150,8 +150,7 @@ class SubtargetPredicate : Predicate<"Subtarget->" # NAME # "()">;
def hasAtomAddF64 : SubtargetPredicate;
def hasAtomScope : SubtargetPredicate;
-def hasAtomBitwise64 : SubtargetPredicate;
-def hasAtomMinMax64 : SubtargetPredicate;
+def hasAtomMinMaxAndOrXor : SubtargetPredicate;
def hasAtomSwap128 : SubtargetPredicate;
def hasClusters : SubtargetPredicate;
def hasPTXASUnreachableBug : SubtargetPredicate;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index ee8adec2da060..357c13a0a679f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -87,8 +87,7 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
}
bool hasAtomAddF64() const { return hasFeature(NVPTX::SM60); }
bool hasAtomScope() const { return hasFeature(NVPTX::SM60); }
- bool hasAtomBitwise64() const { return hasFeature(NVPTX::SM32); }
- bool hasAtomMinMax64() const { return hasFeature(NVPTX::SM32); }
+ bool hasAtomMinMaxAndOrXor() const { return hasFeature(NVPTX::SM32); }
bool hasAtomCas16() const {
return hasFeature(NVPTX::SM70) && hasFeature(NVPTX::PTX63);
}
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/atomicrmw-i64-sm20.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/atomicrmw-i64-sm20.ll
new file mode 100644
index 0000000000000..203cfbf89d1d3
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/atomicrmw-i64-sm20.ll
@@ -0,0 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_20 -mattr=+ptx32 | FileCheck %s --check-prefix=SM20
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_32 -mattr=+ptx41 | FileCheck %s --check-prefix=SM32
+
+target triple = "nvptx64-nvidia-cuda"
+
+define i64 @add_sub_xchg(ptr addrspace(1) %addr, i64 %val) {
+; SM20-LABEL: define i64 @add_sub_xchg(
+; SM20-SAME: ptr addrspace(1) [[ADDR:%.*]], i64 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; SM20-NEXT: [[ADD:%.*]] = atomicrmw add ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM20-NEXT: [[SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM20-NEXT: [[XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM20-NEXT: ret i64 [[XCHG]]
+;
+; SM32-LABEL: define i64 @add_sub_xchg(
+; SM32-SAME: ptr addrspace(1) [[ADDR:%.*]], i64 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; SM32-NEXT: [[ADD:%.*]] = atomicrmw add ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: ret i64 [[XCHG]]
+;
+ %add = atomicrmw add ptr addrspace(1) %addr, i64 %val monotonic
+ %sub = atomicrmw sub ptr addrspace(1) %addr, i64 %val monotonic
+ %xchg = atomicrmw xchg ptr addrspace(1) %addr, i64 %val monotonic
+ ret i64 %xchg
+}
+
+define i64 @minmax_and_or_xor(ptr addrspace(1) %addr, i64 %val) {
+; SM20-LABEL: define i64 @minmax_and_or_xor(
+; SM20-SAME: ptr addrspace(1) [[ADDR:%.*]], i64 [[VAL:%.*]]) #[[ATTR0]] {
+; SM20-NEXT: [[TMP1:%.*]] = load atomic i64, ptr addrspace(1) [[ADDR]] monotonic, align 8
+; SM20-NEXT: br label %[[ATOMICRMW_START32:.*]]
+; SM20: [[ATOMICRMW_START32]]:
+; SM20-NEXT: [[LOADED33:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED36:%.*]], %[[ATOMICRMW_START32]] ]
+; SM20-NEXT: [[NEW34:%.*]] = and i64 [[LOADED33]], [[VAL]]
+; SM20-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i64 [[LOADED33]], i64 [[NEW34]] monotonic monotonic, align 8
+; SM20-NEXT: [[SUCCESS35:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; SM20-NEXT: [[NEWLOADED36]] = extractvalue { i64, i1 } [[TMP2]], 0
+; SM20-NEXT: br i1 [[SUCCESS35]], label %[[ATOMICRMW_END31:.*]], label %[[ATOMICRMW_START32]]
+; SM20: [[ATOMICRMW_END31]]:
+; SM20-NEXT: [[TMP3:%.*]] = load atomic i64, ptr addrspace(1) [[ADDR]] monotonic, align 8
+; SM20-NEXT: br label %[[ATOMICRMW_START26:.*]]
+; SM20: [[ATOMICRMW_START26]]:
+; SM20-NEXT: [[LOADED27:%.*]] = phi i64 [ [[TMP3]], %[[ATOMICRMW_END31]] ], [ [[NEWLOADED30:%.*]], %[[ATOMICRMW_START26]] ]
+; SM20-NEXT: [[NEW28:%.*]] = or i64 [[LOADED27]], [[VAL]]
+; SM20-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i64 [[LOADED27]], i64 [[NEW28]] monotonic monotonic, align 8
+; SM20-NEXT: [[SUCCESS29:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; SM20-NEXT: [[NEWLOADED30]] = extractvalue { i64, i1 } [[TMP4]], 0
+; SM20-NEXT: br i1 [[SUCCESS29]], label %[[ATOMICRMW_END25:.*]], label %[[ATOMICRMW_START26]]
+; SM20: [[ATOMICRMW_END25]]:
+; SM20-NEXT: [[TMP5:%.*]] = load atomic i64, ptr addrspace(1) [[ADDR]] monotonic, align 8
+; SM20-NEXT: br label %[[ATOMICRMW_START20:.*]]
+; SM20: [[ATOMICRMW_START20]]:
+; SM20-NEXT: [[LOADED21:%.*]] = phi i64 [ [[TMP5]], %[[ATOMICRMW_END25]] ], [ [[NEWLOADED24:%.*]], %[[ATOMICRMW_START20]] ]
+; SM20-NEXT: [[NEW22:%.*]] = xor i64 [[LOADED21]], [[VAL]]
+; SM20-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i64 [[LOADED21]], i64 [[NEW22]] monotonic monotonic, align 8
+; SM20-NEXT: [[SUCCESS23:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; SM20-NEXT: [[NEWLOADED24]] = extractvalue { i64, i1 } [[TMP6]], 0
+; SM20-NEXT: br i1 [[SUCCESS23]], label %[[ATOMICRMW_END19:.*]], label %[[ATOMICRMW_START20]]
+; SM20: [[ATOMICRMW_END19]]:
+; SM20-NEXT: [[TMP7:%.*]] = load atomic i64, ptr addrspace(1) [[ADDR]] monotonic, align 8
+; SM20-NEXT: br label %[[ATOMICRMW_START14:.*]]
+; SM20: [[ATOMICRMW_START14]]:
+; SM20-NEXT: [[LOADED15:%.*]] = phi i64 [ [[TMP7]], %[[ATOMICRMW_END19]] ], [ [[NEWLOADED18:%.*]], %[[ATOMICRMW_START14]] ]
+; SM20-NEXT: [[TMP8:%.*]] = icmp sgt i64 [[LOADED15]], [[VAL]]
+; SM20-NEXT: [[NEW16:%.*]] = select i1 [[TMP8]], i64 [[LOADED15]], i64 [[VAL]]
+; SM20-NEXT: [[TMP9:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i64 [[LOADED15]], i64 [[NEW16]] monotonic monotonic, align 8
+; SM20-NEXT: [[SUCCESS17:%.*]] = extractvalue { i64, i1 } [[TMP9]], 1
+; SM20-NEXT: [[NEWLOADED18]] = extractvalue { i64, i1 } [[TMP9]], 0
+; SM20-NEXT: br i1 [[SUCCESS17]], label %[[ATOMICRMW_END13:.*]], label %[[ATOMICRMW_START14]]
+; SM20: [[ATOMICRMW_END13]]:
+; SM20-NEXT: [[TMP10:%.*]] = load atomic i64, ptr addrspace(1) [[ADDR]] monotonic, align 8
+; SM20-NEXT: br label %[[ATOMICRMW_START8:.*]]
+; SM20: [[ATOMICRMW_START8]]:
+; SM20-NEXT: [[LOADED9:%.*]] = phi i64 [ [[TMP10]], %[[ATOMICRMW_END13]] ], [ [[NEWLOADED12:%.*]], %[[ATOMICRMW_START8]] ]
+; SM20-NEXT: [[TMP11:%.*]] = icmp sle i64 [[LOADED9]], [[VAL]]
+; SM20-NEXT: [[NEW10:%.*]] = select i1 [[TMP11]], i64 [[LOADED9]], i64 [[VAL]]
+; SM20-NEXT: [[TMP12:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i64 [[LOADED9]], i64 [[NEW10]] monotonic monotonic, align 8
+; SM20-NEXT: [[SUCCESS11:%.*]] = extractvalue { i64, i1 } [[TMP12]], 1
+; SM20-NEXT: [[NEWLOADED12]] = extractvalue { i64, i1 } [[TMP12]], 0
+; SM20-NEXT: br i1 [[SUCCESS11]], label %[[ATOMICRMW_END7:.*]], label %[[ATOMICRMW_START8]]
+; SM20: [[ATOMICRMW_END7]]:
+; SM20-NEXT: [[TMP13:%.*]] = load atomic i64, ptr addrspace(1) [[ADDR]] monotonic, align 8
+; SM20-NEXT: br label %[[ATOMICRMW_START2:.*]]
+; SM20: [[ATOMICRMW_START2]]:
+; SM20-NEXT: [[LOADED3:%.*]] = phi i64 [ [[TMP13]], %[[ATOMICRMW_END7]] ], [ [[NEWLOADED6:%.*]], %[[ATOMICRMW_START2]] ]
+; SM20-NEXT: [[TMP14:%.*]] = icmp ugt i64 [[LOADED3]], [[VAL]]
+; SM20-NEXT: [[NEW4:%.*]] = select i1 [[TMP14]], i64 [[LOADED3]], i64 [[VAL]]
+; SM20-NEXT: [[TMP15:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i64 [[LOADED3]], i64 [[NEW4]] monotonic monotonic, align 8
+; SM20-NEXT: [[SUCCESS5:%.*]] = extractvalue { i64, i1 } [[TMP15]], 1
+; SM20-NEXT: [[NEWLOADED6]] = extractvalue { i64, i1 } [[TMP15]], 0
+; SM20-NEXT: br i1 [[SUCCESS5]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START2]]
+; SM20: [[ATOMICRMW_END1]]:
+; SM20-NEXT: [[TMP16:%.*]] = load atomic i64, ptr addrspace(1) [[ADDR]] monotonic, align 8
+; SM20-NEXT: br label %[[ATOMICRMW_START:.*]]
+; SM20: [[ATOMICRMW_START]]:
+; SM20-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP16]], %[[ATOMICRMW_END1]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; SM20-NEXT: [[TMP17:%.*]] = icmp ule i64 [[LOADED]], [[VAL]]
+; SM20-NEXT: [[NEW:%.*]] = select i1 [[TMP17]], i64 [[LOADED]], i64 [[VAL]]
+; SM20-NEXT: [[TMP18:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8
+; SM20-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP18]], 1
+; SM20-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP18]], 0
+; SM20-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; SM20: [[ATOMICRMW_END]]:
+; SM20-NEXT: ret i64 [[NEWLOADED]]
+;
+; SM32-LABEL: define i64 @minmax_and_or_xor(
+; SM32-SAME: ptr addrspace(1) [[ADDR:%.*]], i64 [[VAL:%.*]]) #[[ATTR0]] {
+; SM32-NEXT: [[AND:%.*]] = atomicrmw and ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[OR:%.*]] = atomicrmw or ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[MAX:%.*]] = atomicrmw max ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[MIN:%.*]] = atomicrmw min ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: [[UMIN:%.*]] = atomicrmw umin ptr addrspace(1) [[ADDR]], i64 [[VAL]] monotonic, align 8
+; SM32-NEXT: ret i64 [[UMIN]]
+;
+ %and = atomicrmw and ptr addrspace(1) %addr, i64 %val monotonic
+ %or = atomicrmw or ptr addrspace(1) %addr, i64 %val monotonic
+ %xor = atomicrmw xor ptr addrspace(1) %addr, i64 %val monotonic
+ %max = atomicrmw max ptr addrspace(1) %addr, i64 %val monotonic
+ %min = atomicrmw min ptr addrspace(1) %addr, i64 %val monotonic
+ %umax = atomicrmw umax ptr addrspace(1) %addr, i64 %val monotonic
+ %umin = atomicrmw umin ptr addrspace(1) %addr, i64 %val monotonic
+ ret i64 %umin
+}
More information about the llvm-commits
mailing list