[llvm] eb52809 - [NVPTX] Fixup AutoUpgrade of llvm.nvvm.atomic.load.{inc,dec}.32 (#138907)

[via llvm-commits]

Author: Alex MacLean
Date: 2025-05-08T07:17:32-07:00
New Revision: eb5280938bf282053e1b3d281d45b522f1cdcc2e

URL: https://github.com/llvm/llvm-project/commit/eb5280938bf282053e1b3d281d45b522f1cdcc2e
DIFF: https://github.com/llvm/llvm-project/commit/eb5280938bf282053e1b3d281d45b522f1cdcc2e.diff

LOG: [NVPTX] Fixup AutoUpgrade of llvm.nvvm.atomic.load.{inc,dec}.32 (#138907)

The previous implementation failed to account for the fact that these
intrinsics have an overloaded pointer type. This version handles the
pointer type and adds tests for llvm.nvvm.atomic.load.add.{f32,f64}.

Added: 
    

Modified: 
    llvm/lib/IR/AutoUpgrade.cpp
    llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 8bf93555b1fd2..8717cd092b0b5 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1353,12 +1353,15 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         // nvvm.{min,max}.{i,ii,ui,ull}
         Expand = Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
                  Name == "ui" || Name == "ull";
-      else if (Name.consume_front("atomic.load.add."))
-        // nvvm.atomic.load.add.{f32.p,f64.p}
-        Expand = Name.starts_with("f32.p") || Name.starts_with("f64.p");
-      else if (Name.consume_front("atomic.load.") && Name.consume_back(".32"))
-        // nvvm.atomic.load.{inc,dec}.32
-        Expand = Name == "inc" || Name == "dec";
+      else if (Name.consume_front("atomic.load."))
+        // nvvm.atomic.load.add.{f32,f64}.p
+        // nvvm.atomic.load.{inc,dec}.32.p
+        Expand = StringSwitch<bool>(Name)
+                     .StartsWith("add.f32.p", true)
+                     .StartsWith("add.f64.p", true)
+                     .StartsWith("inc.32.p", true)
+                     .StartsWith("dec.32.p", true)
+                     .Default(false);
       else if (Name.consume_front("bitcast."))
         // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
         Expand =
@@ -2383,10 +2386,12 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
     Value *Val = CI->getArgOperand(1);
     Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
                                   AtomicOrdering::SequentiallyConsistent);
-  } else if (Name.consume_front("atomic.load.") && Name.consume_back(".32")) {
+  } else if (Name.starts_with("atomic.load.inc.32.p") ||
+             Name.starts_with("atomic.load.dec.32.p")) {
     Value *Ptr = CI->getArgOperand(0);
     Value *Val = CI->getArgOperand(1);
-    auto Op = Name == "inc" ? AtomicRMWInst::UIncWrap : AtomicRMWInst::UDecWrap;
+    auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
+                                                  : AtomicRMWInst::UDecWrap;
     Rep = Builder.CreateAtomicRMW(Op, Ptr, Val, MaybeAlign(),
                                   AtomicOrdering::SequentiallyConsistent);
   } else if (Name.consume_front("max.") &&

diff  --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 98ffa23fae64b..2bfa1c2dfba7a 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -58,8 +58,10 @@ declare i32 @llvm.nvvm.ldg.global.i.i32.p0(ptr, i32)
 declare ptr @llvm.nvvm.ldg.global.p.p0(ptr, i32)
 declare float @llvm.nvvm.ldg.global.f.f32.p0(ptr, i32)
 
-declare i32 @llvm.nvvm.atomic.load.inc.32(ptr, i32)
-declare i32 @llvm.nvvm.atomic.load.dec.32(ptr, i32)
+declare i32 @llvm.nvvm.atomic.load.inc.32.p0(ptr, i32)
+declare i32 @llvm.nvvm.atomic.load.dec.32.p0(ptr, i32)
+declare i32 @llvm.nvvm.atomic.load.add.f32.p0(ptr, float)
+declare i32 @llvm.nvvm.atomic.load.add.f64.p0(ptr, double)
 
 declare ptr addrspace(3) @llvm.nvvm.mapa.shared.cluster(ptr addrspace(3), i32)
 
@@ -267,12 +269,16 @@ define void @ldg(ptr %p0, ptr addrspace(1) %p1) {
 }
 
 ; CHECK-LABEL: @atomics
-define i32 @atomics(ptr %p0, i32 %a) {
+define i32 @atomics(ptr %p0, i32 %a, float %b, double %c) {
 ; CHECK: %1 = atomicrmw uinc_wrap ptr %p0, i32 %a seq_cst
 ; CHECK: %2 = atomicrmw udec_wrap ptr %p0, i32 %a seq_cst
+; CHECK: %3 = atomicrmw fadd ptr %p0, float %b seq_cst
+; CHECK: %4 = atomicrmw fadd ptr %p0, double %c seq_cst
 
-  %r1 = call i32 @llvm.nvvm.atomic.load.inc.32(ptr %p0, i32 %a)
-  %r2 = call i32 @llvm.nvvm.atomic.load.dec.32(ptr %p0, i32 %a)
+  %r1 = call i32 @llvm.nvvm.atomic.load.inc.32.p0(ptr %p0, i32 %a)
+  %r2 = call i32 @llvm.nvvm.atomic.load.dec.32.p0(ptr %p0, i32 %a)
+  %r3 = call float @llvm.nvvm.atomic.load.add.f32.p0(ptr %p0, float %b)
+  %r4 = call double @llvm.nvvm.atomic.load.add.f64.p0(ptr %p0, double %c)
   ret i32 %r2
 }