[Mlir-commits] [flang] [llvm] [mlir] [openmp][mlir] Expand use of atomicrmw for atomic update (PR #215654)
Ville-Markus Yli-Suutala
llvmlistbot at llvm.org
Mon Oct 19 12:56:53 PDT 2026
https://github.com/VeeEM updated https://github.com/llvm/llvm-project/pull/215654
>From 0e17d4113eb93e85cabd5fd73c40a807f6fd9910 Mon Sep 17 00:00:00 2001
From: Ville-Markus Yli-Suutala <ville-markus.yli-suutala at helsinki.fi>
Date: Sun, 9 Aug 2026 13:42:43 +0300
Subject: [PATCH 1/2] [openmp][mlir] Expand use of atomicrmw for atomic update
Use atomicrmw for single and double precision floats in the following
operations:
- fadd
- fsub
- smax
- smin
- umax
- umin
- fmax
- fmin
- fmaximum
- fminimum
---
.../OpenMP/atomic-capture-release.f90 | 2 +-
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 34 ++-
.../Frontend/OpenMPIRBuilderTest.cpp | 39 +---
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 8 +
mlir/test/Target/LLVMIR/openmp-llvm.mlir | 212 +++++++++++-------
5 files changed, 171 insertions(+), 124 deletions(-)
diff --git a/flang/test/Integration/OpenMP/atomic-capture-release.f90 b/flang/test/Integration/OpenMP/atomic-capture-release.f90
index d0090f7e6ee02..2a3c074539d15 100644
--- a/flang/test/Integration/OpenMP/atomic-capture-release.f90
+++ b/flang/test/Integration/OpenMP/atomic-capture-release.f90
@@ -24,7 +24,7 @@ subroutine test_capture_release(a,b,c)
real(4) :: a, b, c
!$omp atomic capture release
c = a
- a = a + b
+ a = a * b
!$omp end atomic
end subroutine
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 24d8169ffd663..254b47f522984 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -11362,9 +11362,6 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
"OMP atomic update expected a scalar or struct type");
- assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
- (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&
- "OpenMP atomic does not support LT or GT operations");
});
Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
@@ -11392,18 +11389,28 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
return Builder.CreateOr(Src1, Src2);
case AtomicRMWInst::Xor:
return Builder.CreateXor(Src1, Src2);
- case AtomicRMWInst::Xchg:
- case AtomicRMWInst::FAdd:
- case AtomicRMWInst::FSub:
- case AtomicRMWInst::BAD_BINOP:
case AtomicRMWInst::Max:
+ return Builder.CreateBinaryIntrinsic(Intrinsic::smax, Src1, Src2);
case AtomicRMWInst::Min:
+ return Builder.CreateBinaryIntrinsic(Intrinsic::smin, Src1, Src2);
case AtomicRMWInst::UMax:
+ return Builder.CreateBinaryIntrinsic(Intrinsic::umax, Src1, Src2);
case AtomicRMWInst::UMin:
+ return Builder.CreateBinaryIntrinsic(Intrinsic::umin, Src1, Src2);
+ case AtomicRMWInst::FAdd:
+ return Builder.CreateFAdd(Src1, Src2);
+ case AtomicRMWInst::FSub:
+ return Builder.CreateFSub(Src1, Src2);
case AtomicRMWInst::FMax:
+ return Builder.CreateMaxNum(Src1, Src2);
case AtomicRMWInst::FMin:
+ return Builder.CreateMinNum(Src1, Src2);
case AtomicRMWInst::FMaximum:
+ return Builder.CreateMaximum(Src1, Src2);
case AtomicRMWInst::FMinimum:
+ return Builder.CreateMinimum(Src1, Src2);
+ case AtomicRMWInst::Xchg:
+ case AtomicRMWInst::BAD_BINOP:
case AtomicRMWInst::FMaximumNum:
case AtomicRMWInst::FMinimumNum:
case AtomicRMWInst::UIncWrap:
@@ -11444,15 +11451,26 @@ Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor:
case AtomicRMWInst::Xchg:
+ case AtomicRMWInst::Max:
+ case AtomicRMWInst::Min:
+ case AtomicRMWInst::UMax:
+ case AtomicRMWInst::UMin:
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FMax:
+ case AtomicRMWInst::FMin:
+ case AtomicRMWInst::FMaximum:
+ case AtomicRMWInst::FMinimum:
emitRMWOp = XElemTy;
break;
case AtomicRMWInst::Sub:
+ case AtomicRMWInst::FSub:
emitRMWOp = (IsXBinopExpr && XElemTy);
break;
default:
emitRMWOp = false;
}
- emitRMWOp &= XElemTy->isIntegerTy();
+ emitRMWOp &=
+ (XElemTy->isIntegerTy() || XElemTy->isFloatTy() || XElemTy->isDoubleTy());
std::pair<Value *, Value *> Res;
if (emitRMWOp) {
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 084dcb0a5847f..7d02ffe79837a 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -4131,7 +4131,8 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
Type *IntTy = Type::getInt32Ty(M->getContext());
AllocaInst *XVal = Builder.CreateAlloca(IntTy);
XVal->setName("AtomicVar");
- Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0), XVal);
+ StoreInst *Init =
+ Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0), XVal);
OpenMPIRBuilder::AtomicOpValue X = {XVal, IntTy, false, false};
AtomicOrdering AO = AtomicOrdering::Monotonic;
Constant *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
@@ -4153,36 +4154,12 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
AO, RMWOp, UpdateOp,
IsXLHSInRHSPart));
Builder.restoreIP(AfterIP);
- BasicBlock *ContBB = EntryBB->getSingleSuccessor();
- CondBrInst *ContTI = dyn_cast<CondBrInst>(ContBB->getTerminator());
- EXPECT_NE(ContTI, nullptr);
- BasicBlock *EndBB = ContTI->getSuccessor(0);
- EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
- EXPECT_NE(EndBB, nullptr);
-
- PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
- EXPECT_NE(Phi, nullptr);
- EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
- EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
- EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
-
- EXPECT_TRUE(Sub->hasOneUse());
- StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
- AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
-
- ExtractValueInst *ExVI1 =
- dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
- EXPECT_NE(ExVI1, nullptr);
- AtomicCmpXchgInst *CmpExchg =
- dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
- EXPECT_NE(CmpExchg, nullptr);
- EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
- EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
- EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
-
- LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
- EXPECT_NE(Ld, nullptr);
- EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
+ EXPECT_EQ(EntryBB->getParent()->size(), 1U);
+ AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode());
+ EXPECT_NE(ARWM, nullptr);
+ EXPECT_EQ(ARWM->getPointerOperand(), XVal);
+ EXPECT_EQ(ARWM->getOperation(), RMWOp);
+ EXPECT_EQ(ARWM->getValOperand(), Expr);
Builder.CreateRetVoid();
OMPBuilder.finalize();
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 1a861767b5512..cd567c4cd5065 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5701,10 +5701,18 @@ static llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op) {
.Case([&](LLVM::AndOp) { return llvm::AtomicRMWInst::BinOp::And; })
.Case([&](LLVM::OrOp) { return llvm::AtomicRMWInst::BinOp::Or; })
.Case([&](LLVM::XOrOp) { return llvm::AtomicRMWInst::BinOp::Xor; })
+ .Case([&](LLVM::SMaxOp) { return llvm::AtomicRMWInst::BinOp::Max; })
+ .Case([&](LLVM::SMinOp) { return llvm::AtomicRMWInst::BinOp::Min; })
.Case([&](LLVM::UMaxOp) { return llvm::AtomicRMWInst::BinOp::UMax; })
.Case([&](LLVM::UMinOp) { return llvm::AtomicRMWInst::BinOp::UMin; })
.Case([&](LLVM::FAddOp) { return llvm::AtomicRMWInst::BinOp::FAdd; })
.Case([&](LLVM::FSubOp) { return llvm::AtomicRMWInst::BinOp::FSub; })
+ .Case([&](LLVM::MaxNumOp) { return llvm::AtomicRMWInst::BinOp::FMax; })
+ .Case([&](LLVM::MinNumOp) { return llvm::AtomicRMWInst::BinOp::FMin; })
+ .Case(
+ [&](LLVM::MaximumOp) { return llvm::AtomicRMWInst::BinOp::FMaximum; })
+ .Case(
+ [&](LLVM::MinimumOp) { return llvm::AtomicRMWInst::BinOp::FMinimum; })
.Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);
}
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 5fdb7aa72b833..20acdda17ad6c 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1860,12 +1860,12 @@ llvm.func @atomic_update_float_multi_step(%x: !llvm.ptr, %a: f32, %b: f32) {
// CHECK: %[[load:.*]] = load atomic i32, ptr %[[x]] monotonic
// CHECK: %[[phi:.*]] = phi i32
// CHECK: %[[fltCast:.*]] = bitcast i32 %[[phi]] to float
-// CHECK: %[[res:.*]] = call float @llvm.maxnum.f32(float %[[fltCast]], float %[[expr]])
+// CHECK: %[[res:.*]] = call float @llvm.fmuladd.f32(float %[[fltCast]], float %[[expr]], float %[[expr]])
// CHECK: cmpxchg ptr %[[x]], i32 %[[phi]], i32 %{{.*}} monotonic monotonic
llvm.func @atomic_update_float_intrinsic(%x: !llvm.ptr, %expr: f32) {
omp.atomic.update %x : !llvm.ptr {
^bb0(%xval: f32):
- %newval = "llvm.intr.maxnum"(%xval, %expr) : (f32, f32) -> f32
+ %newval = "llvm.intr.fmuladd"(%xval, %expr, %expr) : (f32, f32, f32) -> f32
omp.yield(%newval : f32)
}
llvm.return
@@ -1999,19 +1999,15 @@ llvm.func @omp_atomic_update_ordering(%x:!llvm.ptr, %expr: i32) {
// CHECK-LABEL: @omp_atomic_update_intrinsic
// CHECK-SAME: (ptr %[[x:.*]], i32 %[[expr:.*]])
llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
- // CHECK: %[[t1:.*]] = call i32 @llvm.smax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
- // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
- // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
- // CHECK: cmpxchg ptr %[[x]], i32 %[[x_old]], i32 %[[t2]]
+ // CHECK: %[[res:.*]] = atomicrmw max ptr %[[x]], i32 %[[expr]]
+ // CHECK: %[[newval:.*]] = call i32 @llvm.smax.i32(i32 %[[res]], i32 %[[expr]])
omp.atomic.update %x : !llvm.ptr {
^bb0(%xval: i32):
%newval = "llvm.intr.smax"(%xval, %expr) : (i32, i32) -> i32
omp.yield(%newval : i32)
}
- // CHECK: %[[t1:.*]] = call i32 @llvm.umax.i32(i32 %[[x_old:.*]], i32 %[[expr]])
- // CHECK: store i32 %[[t1]], ptr %[[x_new:.*]]
- // CHECK: %[[t2:.*]] = load i32, ptr %[[x_new]]
- // CHECK: cmpxchg ptr %[[x]], i32 %[[x_old]], i32 %[[t2]]
+ // CHECK: %[[res:.*]] = atomicrmw umax ptr %[[x]], i32 %[[expr]]
+ // CHECK: %[[newval:.*]] = call i32 @llvm.umax.i32(i32 %[[res]], i32 %[[expr]])
omp.atomic.update %x : !llvm.ptr {
^bb0(%xval: i32):
%newval = "llvm.intr.umax"(%xval, %expr) : (i32, i32) -> i32
@@ -2199,11 +2195,8 @@ llvm.func @omp_atomic_capture_prefix_update(
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smax.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
+ // CHECK: %[[res:.*]] = atomicrmw max ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smax.i32(i32 %[[res]], i32 %[[expr]])
// CHECK: store i32 %[[newval]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.update %x : !llvm.ptr {
@@ -2214,11 +2207,8 @@ llvm.func @omp_atomic_capture_prefix_update(
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smin.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
+ // CHECK: %[[res:.*]] = atomicrmw min ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smin.i32(i32 %[[res]], i32 %[[expr]])
// CHECK: store i32 %[[newval]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.update %x : !llvm.ptr {
@@ -2229,11 +2219,8 @@ llvm.func @omp_atomic_capture_prefix_update(
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umax.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
+ // CHECK: %[[res:.*]] = atomicrmw umax ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umax.i32(i32 %[[res]], i32 %[[expr]])
// CHECK: store i32 %[[newval]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.update %x : !llvm.ptr {
@@ -2244,11 +2231,8 @@ llvm.func @omp_atomic_capture_prefix_update(
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umin.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
+ // CHECK: %[[res:.*]] = atomicrmw umin ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umin.i32(i32 %[[res]], i32 %[[expr]])
// CHECK: store i32 %[[newval]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.update %x : !llvm.ptr {
@@ -2259,11 +2243,8 @@ llvm.func @omp_atomic_capture_prefix_update(
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK: %[[newval:.*]] = fadd float %{{.*}}, %[[exprf]]
- // CHECK: store float %[[newval]], ptr %{{.*}}
- // CHECK: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK: %{{.*}} = cmpxchg ptr %[[xf]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
+ // CHECK: %[[res:.*]] = atomicrmw fadd ptr %[[xf]], float %[[exprf]] monotonic
+ // CHECK-NEXT: %[[newval:.*]] = fadd float %[[res]], %[[exprf]]
// CHECK: store float %[[newval]], ptr %[[vf]]
omp.atomic.capture {
omp.atomic.update %xf : !llvm.ptr {
@@ -2274,11 +2255,8 @@ llvm.func @omp_atomic_capture_prefix_update(
omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK: %[[newval:.*]] = fsub float %{{.*}}, %[[exprf]]
- // CHECK: store float %[[newval]], ptr %{{.*}}
- // CHECK: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK: %{{.*}} = cmpxchg ptr %[[xf]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
+ // CHECK: %[[res:.*]] = atomicrmw fsub ptr %[[xf]], float %[[exprf]] monotonic
+ // CHECK-NEXT: %[[newval:.*]] = fsub float %[[res]], %[[exprf]]
// CHECK: store float %[[newval]], ptr %[[vf]]
omp.atomic.capture {
omp.atomic.update %xf : !llvm.ptr {
@@ -2289,6 +2267,54 @@ llvm.func @omp_atomic_capture_prefix_update(
omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
}
+ // CHECK: %[[res:.*]] = atomicrmw fmax ptr %[[xf]], float %[[exprf]]
+ // CHECK-NEXT: %[[newval:.*]] = call float @llvm.maxnum.f32(float %[[res]], float %[[exprf]])
+ // CHECK: store float %[[newval]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.maxnum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ }
+
+ // CHECK: %[[res:.*]] = atomicrmw fmin ptr %[[xf]], float %[[exprf]]
+ // CHECK-NEXT: %[[newval:.*]] = call float @llvm.minnum.f32(float %[[res]], float %[[exprf]])
+ // CHECK: store float %[[newval]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.minnum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ }
+
+ // CHECK: %[[res:.*]] = atomicrmw fmaximum ptr %[[xf]], float %[[exprf]]
+ // CHECK-NEXT: %[[newval:.*]] = call float @llvm.maximum.f32(float %[[res]], float %[[exprf]])
+ // CHECK: store float %[[newval]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.maximum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ }
+
+ // CHECK: %[[res:.*]] = atomicrmw fminimum ptr %[[xf]], float %[[exprf]]
+ // CHECK-NEXT: %[[newval:.*]] = call float @llvm.minimum.f32(float %[[res]], float %[[exprf]])
+ // CHECK: store float %[[newval]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.minimum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ }
+
llvm.return
}
@@ -2444,12 +2470,9 @@ llvm.func @omp_atomic_capture_postfix_update(
}
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smax.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
- // CHECK: store i32 %[[xval]], ptr %[[v]]
+ // CHECK: %[[res:.*]] = atomicrmw max ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smax.i32(i32 %[[res]], i32 %[[expr]])
+ // CHECK: store i32 %[[res]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
omp.atomic.update %x : !llvm.ptr {
@@ -2459,12 +2482,9 @@ llvm.func @omp_atomic_capture_postfix_update(
}
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smin.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
- // CHECK: store i32 %[[xval]], ptr %[[v]]
+ // CHECK: %[[res:.*]] = atomicrmw min ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.smin.i32(i32 %[[res]], i32 %[[expr]])
+ // CHECK: store i32 %[[res]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
omp.atomic.update %x : !llvm.ptr {
@@ -2474,12 +2494,9 @@ llvm.func @omp_atomic_capture_postfix_update(
}
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umax.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
- // CHECK: store i32 %[[xval]], ptr %[[v]]
+ // CHECK: %[[res:.*]] = atomicrmw umax ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umax.i32(i32 %[[res]], i32 %[[expr]])
+ // CHECK: store i32 %[[res]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
omp.atomic.update %x : !llvm.ptr {
@@ -2489,12 +2506,9 @@ llvm.func @omp_atomic_capture_postfix_update(
}
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umin.i32(i32 %[[xval]], i32 %[[expr]])
- // CHECK-NEXT: store i32 %[[newval]], ptr %{{.*}}
- // CHECK-NEXT: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK-NEXT: %{{.*}} = cmpxchg ptr %[[x]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
- // CHECK: store i32 %[[xval]], ptr %[[v]]
+ // CHECK: %[[res:.*]] = atomicrmw umin ptr %[[x]], i32 %[[expr]]
+ // CHECK-NEXT: %[[newval:.*]] = call i32 @llvm.umin.i32(i32 %[[res]], i32 %[[expr]])
+ // CHECK: store i32 %[[res]], ptr %[[v]]
omp.atomic.capture {
omp.atomic.read %v = %x : !llvm.ptr, !llvm.ptr, i32
omp.atomic.update %x : !llvm.ptr {
@@ -2504,13 +2518,8 @@ llvm.func @omp_atomic_capture_postfix_update(
}
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK: %[[xvalf:.*]] = bitcast i32 %[[xval]] to float
- // CHECK: %[[newval:.*]] = fadd float %{{.*}}, %[[exprf]]
- // CHECK: store float %[[newval]], ptr %{{.*}}
- // CHECK: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK: %{{.*}} = cmpxchg ptr %[[xf]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
- // CHECK: store float %[[xvalf]], ptr %[[vf]]
+ // CHECK: %[[res:.*]] = atomicrmw fadd ptr %[[xf]], float %[[exprf]] monotonic
+ // CHECK: store float %[[res]], ptr %[[vf]]
omp.atomic.capture {
omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
omp.atomic.update %xf : !llvm.ptr {
@@ -2520,13 +2529,8 @@ llvm.func @omp_atomic_capture_postfix_update(
}
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK: %[[xvalf:.*]] = bitcast i32 %[[xval]] to float
- // CHECK: %[[newval:.*]] = fsub float %{{.*}}, %[[exprf]]
- // CHECK: store float %[[newval]], ptr %{{.*}}
- // CHECK: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK: %{{.*}} = cmpxchg ptr %[[xf]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
- // CHECK: store float %[[xvalf]], ptr %[[vf]]
+ // CHECK: %[[res:.*]] = atomicrmw fsub ptr %[[xf]], float %[[exprf]] monotonic
+ // CHECK: store float %[[res]], ptr %[[vf]]
omp.atomic.capture {
omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
omp.atomic.update %xf : !llvm.ptr {
@@ -2536,6 +2540,50 @@ llvm.func @omp_atomic_capture_postfix_update(
}
}
+ // CHECK: %[[res:.*]] = atomicrmw fmax ptr %[[xf]], float %[[exprf]]
+ // CHECK: store float %[[res]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.maxnum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ }
+
+ // CHECK: %[[res:.*]] = atomicrmw fmin ptr %[[xf]], float %[[exprf]]
+ // CHECK: store float %[[res]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.minnum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ }
+
+ // CHECK: %[[res:.*]] = atomicrmw fmaximum ptr %[[xf]], float %[[exprf]]
+ // CHECK: store float %[[res]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.maximum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ }
+
+ // CHECK: %[[res:.*]] = atomicrmw fminimum ptr %[[xf]], float %[[exprf]]
+ // CHECK: store float %[[res]], ptr %[[vf]]
+ omp.atomic.capture {
+ omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
+ omp.atomic.update %xf : !llvm.ptr {
+ ^bb0(%xval: f32):
+ %newval = "llvm.intr.minimum"(%xval, %exprf) : (f32, f32) -> f32
+ omp.yield(%newval : f32)
+ }
+ }
+
llvm.return
}
@@ -2552,12 +2600,8 @@ llvm.func @omp_atomic_capture_misc(
omp.atomic.write %x = %expr : !llvm.ptr, i32
}
- // CHECK: %[[xval:.*]] = phi i32
- // CHECK: %[[xvalf:.*]] = bitcast i32 %[[xval]] to float
- // CHECK: store float %[[exprf]], ptr %{{.*}}
- // CHECK: %[[newval_:.*]] = load i32, ptr %{{.*}}
- // CHECK: %{{.*}} = cmpxchg ptr %[[xf]], i32 %[[xval]], i32 %[[newval_]] monotonic monotonic
- // CHECK: store float %[[xvalf]], ptr %[[vf]]
+ // CHECK: %[[xval:.*]] = atomicrmw xchg ptr %[[xf]], float %[[exprf]] monotonic
+ // CHECK: store float %[[xval]], ptr %[[vf]]
omp.atomic.capture{
omp.atomic.read %vf = %xf : !llvm.ptr, !llvm.ptr, f32
omp.atomic.write %xf = %exprf : !llvm.ptr, f32
>From 0c2183792f9590a39e2b3e354eb32f48bc19acfe Mon Sep 17 00:00:00 2001
From: Ville-Markus Yli-Suutala <ville-markus.yli-suutala at helsinki.fi>
Date: Mon, 19 Oct 2026 22:54:26 +0300
Subject: [PATCH 2/2] fixup! [openmp][mlir] Expand use of atomicrmw for atomic
update
---
llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h | 13 ++++++++-----
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 2 --
mlir/test/Target/LLVMIR/openmp-llvm.mlir | 15 +++++++++++++++
3 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 3560cfef096fe..a7812a37a772b 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -3995,7 +3995,8 @@ class OpenMPIRBuilder {
/// instructions.
/// \param RMWOp The binary operation used for update. If
/// operation is not supported by atomicRMW,
- /// or belong to {FADD, FSUB, BAD_BINOP}.
+ /// or belong to {FMaximumNum, FMinimumNum, UDecWrap,
+ /// UIncWrap, USubCond, USubSat, Xchg, BAD_BINOP}.
/// Then a `cmpExch` based atomic will be generated.
/// \param UpdateOp Code generator for complex expressions that cannot be
/// expressed through atomicrmw instruction.
@@ -4072,8 +4073,9 @@ class OpenMPIRBuilder {
/// \param AO Atomic ordering of the generated atomic instructions.
/// \param RMWOp The binary operation used for update. If operation
/// is not supported by atomicRMW, or belong to
- /// {FADD, FSUB, BAD_BINOP}. Then a `cmpExch` based
- /// atomic will be generated.
+ /// {FMaximumNum, FMinimumNum, UDecWrap, UIncWrap,
+ /// USubCond, USubSat, Xchg, BAD_BINOP}. Then a
+ /// `cmpExch` based atomic will be generated.
/// \param UpdateOp Code generator for complex expressions that cannot be
/// expressed through atomicrmw instruction.
/// \param IsXBinopExpr true if \a X is Left H.S. in Right H.S. part of the
@@ -4104,8 +4106,9 @@ class OpenMPIRBuilder {
/// \param AO Atomic ordering of the generated atomic instructions
/// \param RMWOp The binary operation used for update. If
/// operation is not supported by atomicRMW, or belong to
- /// {FADD, FSUB, BAD_BINOP}. Then a cmpExch based
- /// atomic will be generated.
+ /// {FMaximumNum, FMinimumNum, UDecWrap, UIncWrap,
+ /// USubCond, USubSat, Xchg, BAD_BINOP}. Then a
+ /// `cmpExch` based atomic will be generated.
/// \param UpdateOp Code generator for complex expressions that cannot be
/// expressed through atomicrmw instruction.
/// \param UpdateExpr true if X is an in place update of the form
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 254b47f522984..a31bc68224658 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -11632,8 +11632,6 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
"OMP atomic capture expected a scalar or struct type");
- assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
- "OpenMP atomic does not support LT or GT operations");
});
// If UpdateExpr is 'x' updated with some `expr` not based on 'x',
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 20acdda17ad6c..9a5f6f7f5e82c 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1673,6 +1673,21 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
// -----
+// Check a double precision atomic uptade that uses atomicrmw.
+// CHECK-LABEL: @omp_atomic_update
+// CHECK-SAME: (ptr %[[x:.*]], double %[[expr:.*]])
+llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: f64) {
+ // CHECK: atomicrmw fadd ptr %[[x]], double %[[expr]] monotonic
+ omp.atomic.update %x : !llvm.ptr {
+ ^bb0(%xval: f64):
+ %newval = llvm.fadd %xval, %expr : f64
+ omp.yield(%newval : f64)
+ }
+ llvm.return
+}
+
+// -----
+
// CHECK-LABEL: @omp_atomic_write
llvm.func @omp_atomic_write() {
// CHECK: %[[ALLOCA0:.*]] = alloca { float, float }, align 8
More information about the Mlir-commits
mailing list