[Mlir-commits] [mlir] 1491508 - [mlir][LLVM] Add ignore_denormal_mode UnitAttr to LLVM::AtomicRMWOp (#217588)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Mon Oct 12 01:57:41 PDT 2026
Author: Christian Sigg
Date: 2026-10-12T10:57:09+02:00
New Revision: 1491508743104fe76da44e5dadb1b67e1035abcf
URL: https://github.com/llvm/llvm-project/commit/1491508743104fe76da44e5dadb1b67e1035abcf
DIFF: https://github.com/llvm/llvm-project/commit/1491508743104fe76da44e5dadb1b67e1035abcf.diff
LOG: [mlir][LLVM] Add ignore_denormal_mode UnitAttr to LLVM::AtomicRMWOp (#217588)
Add an `ignore_denormal_mode` unit attribute to `llvm.atomicrmw`. It
maps 1:1 to the target-independent `!atomic.ignore.denormal.mode`
instruction metadata (#217585): export attaches the metadata when the
attribute is set, and import sets the attribute when the metadata is
present.
The metadata tells the backend that the atomic may ignore the function's
denormal mode, so a native atomic instruction with fixed denormal
behavior can be used instead of a CAS loop. AMDGPU already honors it,
and NVPTX does as of #217586. Until now MLIR could only produce it on
AMDGPU, through the discardable `rocdl.ignore_denormal_mode` attribute.
That leaves frontends like XLA with no way to request it for NVPTX, and
forces target-specific code for something that is target-independent in
LLVM IR.
The attribute is modeled like `volatile_`: an inherent unit attribute
with an optional `ignore_denormal_mode` keyword in the assembly format
and a trailing `ignoreDenormalMode` builder argument, so existing
builder calls are unaffected:
```mlir
%0 = llvm.atomicrmw ignore_denormal_mode fadd %ptr, %val monotonic : !llvm.ptr, f32
```
The verifier rejects the attribute on non-floating-point operations. To
share the check, the floating-point opcode list from the verifier is
moved into an `isFloatingPointOperation()` helper.
`rocdl.ignore_denormal_mode` is left in place for now so downstream
users can migrate. It will be removed in a follow-up.
Added:
Modified:
mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
mlir/test/Dialect/LLVMIR/invalid.mlir
mlir/test/Dialect/LLVMIR/roundtrip.mlir
mlir/test/Target/LLVMIR/Import/instructions.ll
mlir/test/Target/LLVMIR/llvmir.mlir
Removed:
################################################################################
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index 34fb1c6b3153c..34a47274a7fdf 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -216,6 +216,13 @@ class LLVM_MemOpPatterns {
code setVolatileCode = [{
inst->setVolatile($volatile_);
}];
+ code setIgnoreDenormalModeCode = [{
+ if ($ignore_denormal_mode) {
+ llvm::MDNode *metadata = llvm::MDNode::get(inst->getContext(), {});
+ inst->setMetadata(llvm::LLVMContext::MD_atomic_ignore_denormal_mode,
+ metadata);
+ }
+ }];
code setSyncScopeCode = [{
if ($syncscope.has_value()) {
llvm::LLVMContext &llvmContext = builder.getContext();
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 8dbd6a75cbcd3..b09cdf9000d1e 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -2436,12 +2436,14 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering,
OptionalAttr<StrAttr>:$syncscope,
OptionalAttr<I64Attr>:$alignment,
- UnitAttr:$volatile_);
+ UnitAttr:$volatile_,
+ UnitAttr:$ignore_denormal_mode);
// Append the aliasing related attributes defined in LLVM_MemAccessOpBase.
let arguments = !con(args, aliasAttrs);
let results = (outs LLVM_AtomicRMWType:$res);
let assemblyFormat = [{
- (`volatile` $volatile_^)? $bin_op $ptr `,` $val
+ (`volatile` $volatile_^)? (`ignore_denormal_mode` $ignore_denormal_mode^)?
+ $bin_op $ptr `,` $val
(`syncscope` `(` $syncscope^ `)`)? $ordering
prop-dict
attr-dict `:`
@@ -2454,6 +2456,7 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
convertAtomicOrderingToLLVM($ordering));
$res = inst;
}] # setVolatileCode
+ # setIgnoreDenormalModeCode
# setSyncScopeCode
# setAlignmentCode
# setAccessGroupsMetadataCode
@@ -2464,16 +2467,30 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
$res = LLVM::AtomicRMWOp::create($_builder, $_location,
convertAtomicBinOpFromLLVM(atomicInst->getOperation()), $ptr, $val,
convertAtomicOrderingFromLLVM(atomicInst->getOrdering()),
- getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile());
+ getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile(),
+ atomicInst->hasMetadata(llvm::LLVMContext::MD_atomic_ignore_denormal_mode));
}];
- list<int> llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1];
+ list<int> llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1];
let builders = [
OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val,
"LLVM::AtomicOrdering":$ordering,
CArg<"StringRef", "StringRef()">:$syncscope,
- CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile
+ CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile,
+ CArg<"bool", "false">:$ignoreDenormalMode
)>
];
+ let extraClassDeclaration = [{
+ bool isFloatingPointOperation() {
+ return getBinOp() == AtomicBinOp::fadd ||
+ getBinOp() == AtomicBinOp::fsub ||
+ getBinOp() == AtomicBinOp::fmin ||
+ getBinOp() == AtomicBinOp::fmax ||
+ getBinOp() == AtomicBinOp::fminimum ||
+ getBinOp() == AtomicBinOp::fmaximum ||
+ getBinOp() == AtomicBinOp::fminimumnum ||
+ getBinOp() == AtomicBinOp::fmaximumnum;
+ }
+ }];
let hasVerifier = 1;
}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index f498b5d042a16..be5ea6df0caba 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3848,22 +3848,19 @@ OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
AtomicBinOp binOp, Value ptr, Value val,
AtomicOrdering ordering, StringRef syncscope,
- unsigned alignment, bool isVolatile) {
+ unsigned alignment, bool isVolatile,
+ bool ignoreDenormalMode) {
build(builder, state, val.getType(), binOp, ptr, val, ordering,
!syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
+ ignoreDenormalMode,
/*access_groups=*/nullptr,
/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
LogicalResult AtomicRMWOp::verify() {
auto valType = getVal().getType();
- if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub ||
- getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax ||
- getBinOp() == AtomicBinOp::fminimum ||
- getBinOp() == AtomicBinOp::fmaximum ||
- getBinOp() == AtomicBinOp::fminimumnum ||
- getBinOp() == AtomicBinOp::fmaximumnum) {
+ if (isFloatingPointOperation()) {
if (isCompatibleVectorType(valType)) {
if (isScalableVectorType(valType))
return emitOpError("expected LLVM IR fixed vector type");
@@ -3886,6 +3883,10 @@ LogicalResult AtomicRMWOp::verify() {
return emitOpError("expected LLVM IR integer type");
}
+ if (getIgnoreDenormalMode() && !isFloatingPointOperation())
+ return emitOpError(
+ "expected floating-point operation with 'ignore_denormal_mode'");
+
if (static_cast<unsigned>(getOrdering()) <
static_cast<unsigned>(AtomicOrdering::monotonic))
return emitOpError() << "expected at least '"
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index d1f1d0ad9ffaa..fc2ecd9d11b64 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -998,6 +998,14 @@ func.func @atomicrmw_expected_int(%f32_ptr : !llvm.ptr, %f32 : f32) {
// -----
+func.func @atomicrmw_expected_float_ignore_denormal_mode(%i32_ptr : !llvm.ptr, %i32 : i32) {
+ // expected-error at +1 {{expected floating-point operation with 'ignore_denormal_mode'}}
+ %0 = llvm.atomicrmw ignore_denormal_mode add %i32_ptr, %i32 monotonic : !llvm.ptr, i32
+ llvm.return
+}
+
+// -----
+
func.func @cmpxchg_mismatched_value_operands(%ptr : !llvm.ptr, %i32 : i32, %i64 : i64) {
// expected-error at +1 {{op failed to verify that operand #1 and operand #2 have the same type}}
%0 = "llvm.cmpxchg"(%ptr, %i32, %i64) <{success_ordering=2,failure_ordering=2}> : (!llvm.ptr, i32, i64) -> !llvm.struct<(i32, i1)>
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 688dc37603ba5..8f7e3c8707e31 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -579,6 +579,10 @@ func.func @atomicrmw(%ptr : !llvm.ptr, %f32 : f32, %f16_vec : vector<2xf16>) {
%2 = llvm.atomicrmw fmin %ptr, %f16_vec monotonic : !llvm.ptr, vector<2xf16>
// CHECK: llvm.atomicrmw fminimumnum %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32
%3 = llvm.atomicrmw fminimumnum %ptr, %f32 monotonic : !llvm.ptr, f32
+ // CHECK: llvm.atomicrmw ignore_denormal_mode fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32
+ %4 = llvm.atomicrmw ignore_denormal_mode fadd %ptr, %f32 monotonic : !llvm.ptr, f32
+ // CHECK: llvm.atomicrmw volatile ignore_denormal_mode fsub %{{.*}}, %{{.*}} syncscope("singlethread") monotonic <alignment = 16> : !llvm.ptr, f32
+ %5 = llvm.atomicrmw volatile ignore_denormal_mode fsub %ptr, %f32 syncscope("singlethread") monotonic <alignment = 16> : !llvm.ptr, f32
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll
index 28ab4bc44a40a..e6caf655aa667 100644
--- a/mlir/test/Target/LLVMIR/Import/instructions.ll
+++ b/mlir/test/Target/LLVMIR/Import/instructions.ll
@@ -485,9 +485,13 @@ define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) {
; CHECK-SAME: syncscope("singlethread")
; CHECK-SAME: alignment = 8
%22 = atomicrmw volatile udec_wrap ptr %ptr1, i32 %val1 syncscope("singlethread") acquire, align 8
+ ; CHECK: llvm.atomicrmw ignore_denormal_mode fadd %[[PTR2]], %[[VAL2]] acquire
+ %23 = atomicrmw fadd ptr %ptr2, float %val2 acquire, !atomic.ignore.denormal.mode !0
ret void
}
+!0 = !{}
+
; // -----
; CHECK-LABEL: @atomic_cmpxchg
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index f29184beb6b4d..e5d70b756d488 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -1688,6 +1688,8 @@ llvm.func @atomicrmw(
// CHECK-SAME: syncscope("singlethread")
// CHECK-SAME: align 8
%27 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic <alignment = 8> : !llvm.ptr, i32
+ // CHECK: atomicrmw fadd ptr %{{.*}}, float %{{.*}} monotonic, align 4, !atomic.ignore.denormal.mode !{{[0-9]+}}
+ %28 = llvm.atomicrmw ignore_denormal_mode fadd %f32_ptr, %f32 monotonic : !llvm.ptr, f32
llvm.return
}
More information about the Mlir-commits
mailing list