[Mlir-commits] [mlir] b2317cc - [mlir][arith] Add rounding mode flags to binary arithmetic operations (#188458)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Fri Apr 17 01:13:28 PDT 2026
Author: Matthias Springer
Date: 2026-04-17T09:13:23+01:00
New Revision: b2317cc584b84b275a60004226b5962e635c4fda
URL: https://github.com/llvm/llvm-project/commit/b2317cc584b84b275a60004226b5962e635c4fda
DIFF: https://github.com/llvm/llvm-project/commit/b2317cc584b84b275a60004226b5962e635c4fda.diff
LOG: [mlir][arith] Add rounding mode flags to binary arithmetic operations (#188458)
Add rounding mode flags for `addf`, `subf`, `mulf`, `divf`. This
addresses a TODO in the op description.
The folder now takes into account the specified rounding mode. If no
rounding mode is specified, the folders/canonicalizations default to
`rmNearestTiesToEven`. (This behavior has not changed.) This is
documented in the top-level arith dialect documentation. The default
arith rounding mode applies only to "internal" transformations such as
foldings/canonicalizations. In case of an unspecified explicit rounding
mode, the runtime behavior is up to the target backend.
Also add a lowering to LLVM intrinsics such as
`llvm.intr.experimental.constrained.fadd`.
Assisted-by: claude-4.6-opus-high
Added:
Modified:
mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h
mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
mlir/lib/Dialect/Arith/IR/ArithOps.cpp
mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
mlir/test/Dialect/Arith/canonicalize.mlir
mlir/test/Dialect/Arith/ops.mlir
Removed:
################################################################################
diff --git a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h
index fccfe4897114e..feb74c86e349f 100644
--- a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h
+++ b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h
@@ -157,6 +157,11 @@ class AttrConverterConstrainedFPToLLVM {
convertedAttr.set(TargetOp::getRoundingModeAttrName(),
convertArithRoundingModeAttrToLLVM(arithAttr));
}
+ // Constrained intrinsics (llvm.intr.experimental.constrained.*) do not
+ // support fastmath flags. Remove the arith fastmath attribute if present.
+ if constexpr (SourceOp::template hasTrait<
+ arith::ArithFastMathInterface::Trait>())
+ convertedAttr.erase(srcOp.getFastMathAttrName());
convertedAttr.set(TargetOp::getFPExceptionBehaviorAttrName(),
getLLVMDefaultFPExceptionBehavior(*srcOp->getContext()));
}
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
index d4649c45c90c2..985ae01008002 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
@@ -29,6 +29,13 @@ def Arith_Dialect : Dialect {
Manipulating value with type `i0` isn't supported in this dialect at the
moment and is considered invalid. This can change in the future if some
motivating use-cases are presented.
+
+ Some floating-point operations may specify rounding modes and/or fast-math
+ flags. In the absence of an explicit rounding mode, the arith dialect uses
+ this default round mode for internal purposes such as constant folding and
+ canonicalization: round-to-nearest, ties-to-even. The runtime behavior of
+ operations without an explicit rounding mode is deferred to the target
+ backend and may
diff er from the default arith rounding mode.
}];
let hasConstantMaterializer = 1;
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index 45cb3cecef3d8..e6a29180066e9 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -81,15 +81,52 @@ class Arith_FloatUnaryOp<string mnemonic, list<Trait> traits = []> :
class Arith_FloatBinaryOp<string mnemonic, list<Trait> traits = []> :
Arith_BinaryOp<mnemonic,
!listconcat([Pure, DeclareOpInterfaceMethods<ArithFastMathInterface>],
- traits)>,
- Arguments<(ins FloatLike:$lhs, FloatLike:$rhs,
+ traits)> {
+ let arguments = (ins FloatLike:$lhs, FloatLike:$rhs,
DefaultValuedAttr<
- Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath)>,
- Results<(outs FloatLike:$result)> {
+ Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath);
+ let results = (outs FloatLike:$result);
let assemblyFormat = [{ $lhs `,` $rhs (`fastmath` `` $fastmath^)?
attr-dict `:` type($result) }];
}
+// Base class for floating point binary operations with an optional rounding
+// mode.
+class Arith_FloatBinaryOpWithRoundingMode<string mnemonic,
+ list<Trait> traits = []> :
+ Arith_FloatBinaryOp<mnemonic,
+ !listconcat([DeclareOpInterfaceMethods<ArithRoundingModeInterface>],
+ traits)> {
+ let arguments = (ins FloatLike:$lhs, FloatLike:$rhs,
+ DefaultValuedAttr<
+ Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath,
+ OptionalAttr<Arith_RoundingModeAttr>:$roundingmode);
+ let builders = [
+ OpBuilder<(ins "Value":$lhs, "Value":$rhs,
+ CArg<"::mlir::arith::FastMathFlags",
+ "::mlir::arith::FastMathFlags::none">:$fastmath), [{
+ build($_builder, $_state, lhs, rhs, fastmath,
+ ::mlir::arith::RoundingModeAttr{});
+ }]>,
+ OpBuilder<(ins "Value":$lhs, "Value":$rhs,
+ "::mlir::arith::FastMathFlagsAttr":$fastmath), [{
+ build($_builder, $_state, lhs, rhs, fastmath,
+ ::mlir::arith::RoundingModeAttr{});
+ }]>,
+ OpBuilder<(ins "Type":$type, "Value":$lhs, "Value":$rhs,
+ CArg<"::mlir::arith::FastMathFlags",
+ "::mlir::arith::FastMathFlags::none">:$fastmath), [{
+ build($_builder, $_state, type, lhs, rhs,
+ ::mlir::arith::FastMathFlagsAttr::get(
+ $_builder.getContext(), fastmath),
+ ::mlir::arith::RoundingModeAttr{});
+ }]>,
+ ];
+ let assemblyFormat = [{ $lhs `,` $rhs ($roundingmode^)?
+ (`fastmath` `` $fastmath^)?
+ attr-dict `:` type($result) }];
+}
+
// Checks that tensor input and outputs have identical shapes. This is stricker
// than the verification done in `SameOperandsAndResultShape` that allows for
// tensor dimensions to be 'compatible' (e.g., dynamic dimensions being
@@ -957,7 +994,7 @@ def Arith_NegFOp : Arith_FloatUnaryOp<"negf"> {
// AddFOp
//===----------------------------------------------------------------------===//
-def Arith_AddFOp : Arith_FloatBinaryOp<"addf", [Commutative]> {
+def Arith_AddFOp : Arith_FloatBinaryOpWithRoundingMode<"addf", [Commutative]> {
let summary = "floating point addition operation";
let description = [{
The `addf` operation takes two operands and returns one result, each of
@@ -976,10 +1013,10 @@ def Arith_AddFOp : Arith_FloatBinaryOp<"addf", [Commutative]> {
// Tensor addition.
%x = arith.addf %y, %z : tensor<4x?xbf16>
- ```
- TODO: In the distant future, this will accept optional attributes for fast
- math, contraction, rounding mode, and other controls.
+ // Scalar addition with rounding mode.
+ %a = arith.addf %b, %c to_nearest_even : f64
+ ```
}];
let hasFolder = 1;
}
@@ -988,7 +1025,7 @@ def Arith_AddFOp : Arith_FloatBinaryOp<"addf", [Commutative]> {
// SubFOp
//===----------------------------------------------------------------------===//
-def Arith_SubFOp : Arith_FloatBinaryOp<"subf"> {
+def Arith_SubFOp : Arith_FloatBinaryOpWithRoundingMode<"subf"> {
let summary = "floating point subtraction operation";
let description = [{
The `subf` operation takes two operands and returns one result, each of
@@ -1007,10 +1044,10 @@ def Arith_SubFOp : Arith_FloatBinaryOp<"subf"> {
// Tensor subtraction.
%x = arith.subf %y, %z : tensor<4x?xbf16>
- ```
- TODO: In the distant future, this will accept optional attributes for fast
- math, contraction, rounding mode, and other controls.
+ // Scalar subtraction with rounding mode.
+ %a = arith.subf %b, %c downward : f64
+ ```
}];
let hasFolder = 1;
}
@@ -1139,7 +1176,7 @@ def Arith_MinUIOp : Arith_TotalIntBinaryOp<"minui", [Commutative]> {
// MulFOp
//===----------------------------------------------------------------------===//
-def Arith_MulFOp : Arith_FloatBinaryOp<"mulf", [Commutative]> {
+def Arith_MulFOp : Arith_FloatBinaryOpWithRoundingMode<"mulf", [Commutative]> {
let summary = "floating point multiplication operation";
let description = [{
The `mulf` operation takes two operands and returns one result, each of
@@ -1158,10 +1195,10 @@ def Arith_MulFOp : Arith_FloatBinaryOp<"mulf", [Commutative]> {
// Tensor pointwise multiplication.
%x = arith.mulf %y, %z : tensor<4x?xbf16>
- ```
- TODO: In the distant future, this will accept optional attributes for fast
- math, contraction, rounding mode, and other controls.
+ // Scalar multiplication with rounding mode.
+ %a = arith.mulf %b, %c upward : f64
+ ```
}];
let hasFolder = 1;
let hasCanonicalizer = 1;
@@ -1171,8 +1208,24 @@ def Arith_MulFOp : Arith_FloatBinaryOp<"mulf", [Commutative]> {
// DivFOp
//===----------------------------------------------------------------------===//
-def Arith_DivFOp : Arith_FloatBinaryOp<"divf"> {
+def Arith_DivFOp : Arith_FloatBinaryOpWithRoundingMode<"divf"> {
let summary = "floating point division operation";
+ let description = [{
+ The `divf` operation takes two operands and returns one result, each of
+ these is required to be the same type. This type may be a floating point
+ scalar type, a vector whose element type is a floating point type, or a
+ floating point tensor.
+
+ Example:
+
+ ```mlir
+ // Scalar division.
+ %a = arith.divf %b, %c : f64
+
+ // Scalar division with rounding mode.
+ %a = arith.divf %b, %c toward_zero : f64
+ ```
+ }];
let hasFolder = 1;
let hasCanonicalizer = 1;
}
@@ -1186,6 +1239,8 @@ def Arith_RemFOp : Arith_FloatBinaryOp<"remf"> {
let description = [{
Returns the floating point division remainder.
The remainder has the same sign as the dividend (lhs operand).
+
+ TODO: Add support for rounding modes.
}];
let hasFolder = 1;
}
@@ -1420,8 +1475,6 @@ def Arith_TruncFOp :
let description = [{
Truncate a floating-point value to a smaller floating-point-typed value.
The destination type must be strictly narrower than the source type.
- If the value cannot be exactly represented, it is rounded using the
- provided rounding mode or the default one if no rounding mode is provided.
When operating on vectors, casts elementwise.
}];
let builders = [
@@ -1461,9 +1514,7 @@ def Arith_ConvertFOp :
be represented by `arith.extf` or `arith.truncf`.
The source and destination element types must be
diff erent and must have
- the same bitwidth. If the value cannot be exactly represented, it is
- rounded using the provided rounding mode or the default one if no rounding
- mode is provided. When operating on vectors, casts elementwise.
+ the same bitwidth. When operating on vectors, casts elementwise.
}];
let hasFolder = 1;
@@ -1552,9 +1603,7 @@ def Arith_UIToFPOp :
let summary = "cast from unsigned integer type to floating-point";
let description = [{
Cast from a value interpreted as unsigned integer to the corresponding
- floating-point value. If the value cannot be exactly represented, it is
- rounded using the default rounding mode. When operating on vectors, casts
- elementwise.
+ floating-point value. When operating on vectors, casts elementwise.
When the `nneg` flag is present, the operand is assumed to have
the most significant bit set to 0. In this case, zero extension is
@@ -1589,9 +1638,7 @@ def Arith_SIToFPOp : Arith_IToFCastOp<"sitofp"> {
let summary = "cast from integer type to floating-point";
let description = [{
Cast from a value interpreted as a signed integer to the corresponding
- floating-point value. If the value cannot be exactly represented, it is
- rounded using the default rounding mode. When operating on vectors, casts
- elementwise.
+ floating-point value. When operating on vectors, casts elementwise.
}];
let hasFolder = 1;
let hasCanonicalizer = 1;
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index f9ea8dba105a4..2624420cf5318 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -29,13 +29,20 @@ using namespace mlir;
namespace {
-/// Operations whose conversion will depend on whether they are passed a
-/// rounding mode attribute or not.
+/// Lowering pattern that matches only when the source op's rounding mode
+/// presence agrees with `HasRoundingMode`. This allows registering two
+/// instances of the same pattern for one source op: one that handles the
+/// unconstrained case (no rounding mode, lowering to a regular LLVM op) and
+/// one that handles the constrained case (rounding mode present, lowering to
+/// a constrained LLVM intrinsic).
///
-/// `SourceOp` is the source operation; `TargetOp`, the operation it will lower
-/// to; `AttrConvert` is the attribute conversion to convert the rounding mode
-/// attribute.
-template <typename SourceOp, typename TargetOp, bool Constrained,
+/// * `HasRoundingMode`: the pattern matches if and only if the source op has
+/// a rounding mode attribute.
+/// * `AttrConvert`: attribute converter to translate source attributes to
+/// target attributes.
+/// * `FailOnUnsupportedFP`: whether to fail if the source op has unsupported
+/// floating point types.
+template <typename SourceOp, typename TargetOp, bool HasRoundingMode,
template <typename, typename> typename AttrConvert =
AttrConvertPassThrough,
bool FailOnUnsupportedFP = false>
@@ -49,7 +56,7 @@ struct ConstrainedVectorConvertToLLVMPattern
LogicalResult
matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (Constrained != static_cast<bool>(op.getRoundingModeAttr()))
+ if (HasRoundingMode != static_cast<bool>(op.getRoundingModeAttr()))
return failure();
return VectorConvertToLLVMPattern<
SourceOp, TargetOp, AttrConvert,
@@ -81,9 +88,13 @@ struct IdentityBitcastLowering final
//===----------------------------------------------------------------------===//
using AddFOpLowering =
- VectorConvertToLLVMPattern<arith::AddFOp, LLVM::FAddOp,
- arith::AttrConvertFastMathToLLVM,
- /*FailOnUnsupportedFP=*/true>;
+ ConstrainedVectorConvertToLLVMPattern<arith::AddFOp, LLVM::FAddOp,
+ /*HasRoundingMode=*/false,
+ arith::AttrConvertFastMathToLLVM,
+ /*FailOnUnsupportedFP=*/true>;
+using ConstrainedAddFOpLowering = ConstrainedVectorConvertToLLVMPattern<
+ arith::AddFOp, LLVM::ConstrainedFAddIntr, /*HasRoundingMode=*/true,
+ arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
using AddIOpLowering =
VectorConvertToLLVMPattern<arith::AddIOp, LLVM::AddOp,
arith::AttrConvertOverflowToLLVM>;
@@ -91,9 +102,13 @@ using AndIOpLowering = VectorConvertToLLVMPattern<arith::AndIOp, LLVM::AndOp>;
using BitcastOpLowering =
VectorConvertToLLVMPattern<arith::BitcastOp, LLVM::BitcastOp>;
using DivFOpLowering =
- VectorConvertToLLVMPattern<arith::DivFOp, LLVM::FDivOp,
- arith::AttrConvertFastMathToLLVM,
- /*FailOnUnsupportedFP=*/true>;
+ ConstrainedVectorConvertToLLVMPattern<arith::DivFOp, LLVM::FDivOp,
+ /*HasRoundingMode=*/false,
+ arith::AttrConvertFastMathToLLVM,
+ /*FailOnUnsupportedFP=*/true>;
+using ConstrainedDivFOpLowering = ConstrainedVectorConvertToLLVMPattern<
+ arith::DivFOp, LLVM::ConstrainedFDivIntr, /*HasRoundingMode=*/true,
+ arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
using DivSIOpLowering =
VectorConvertToLLVMPattern<arith::DivSIOp, LLVM::SDivOp>;
using DivUIOpLowering =
@@ -139,9 +154,13 @@ using MinSIOpLowering =
using MinUIOpLowering =
VectorConvertToLLVMPattern<arith::MinUIOp, LLVM::UMinOp>;
using MulFOpLowering =
- VectorConvertToLLVMPattern<arith::MulFOp, LLVM::FMulOp,
- arith::AttrConvertFastMathToLLVM,
- /*FailOnUnsupportedFP=*/true>;
+ ConstrainedVectorConvertToLLVMPattern<arith::MulFOp, LLVM::FMulOp,
+ /*HasRoundingMode=*/false,
+ arith::AttrConvertFastMathToLLVM,
+ /*FailOnUnsupportedFP=*/true>;
+using ConstrainedMulFOpLowering = ConstrainedVectorConvertToLLVMPattern<
+ arith::MulFOp, LLVM::ConstrainedFMulIntr, /*HasRoundingMode=*/true,
+ arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
using MulIOpLowering =
VectorConvertToLLVMPattern<arith::MulIOp, LLVM::MulOp,
arith::AttrConvertOverflowToLLVM>;
@@ -170,18 +189,23 @@ using ShRUIOpLowering =
using SIToFPOpLowering =
VectorConvertToLLVMPattern<arith::SIToFPOp, LLVM::SIToFPOp>;
using SubFOpLowering =
- VectorConvertToLLVMPattern<arith::SubFOp, LLVM::FSubOp,
- arith::AttrConvertFastMathToLLVM,
- /*FailOnUnsupportedFP=*/true>;
+ ConstrainedVectorConvertToLLVMPattern<arith::SubFOp, LLVM::FSubOp,
+ /*HasRoundingMode=*/false,
+ arith::AttrConvertFastMathToLLVM,
+ /*FailOnUnsupportedFP=*/true>;
+using ConstrainedSubFOpLowering = ConstrainedVectorConvertToLLVMPattern<
+ arith::SubFOp, LLVM::ConstrainedFSubIntr, /*HasRoundingMode=*/true,
+ arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
using SubIOpLowering =
VectorConvertToLLVMPattern<arith::SubIOp, LLVM::SubOp,
arith::AttrConvertOverflowToLLVM>;
using TruncFOpLowering =
ConstrainedVectorConvertToLLVMPattern<arith::TruncFOp, LLVM::FPTruncOp,
- false, AttrConvertPassThrough,
+ /*HasRoundingMode=*/false,
+ AttrConvertPassThrough,
/*FailOnUnsupportedFP=*/true>;
using ConstrainedTruncFOpLowering = ConstrainedVectorConvertToLLVMPattern<
- arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, true,
+ arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, /*HasRoundingMode=*/true,
arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
using TruncIOpLowering =
VectorConvertToLLVMPattern<arith::TruncIOp, LLVM::TruncOp,
@@ -700,6 +724,7 @@ void mlir::arith::populateArithToLLVMConversionPatterns(
// clang-format off
patterns.add<
AddFOpLowering,
+ ConstrainedAddFOpLowering,
AddIOpLowering,
AndIOpLowering,
AddUIExtendedOpLowering,
@@ -708,6 +733,7 @@ void mlir::arith::populateArithToLLVMConversionPatterns(
CmpFOpLowering,
CmpIOpLowering,
DivFOpLowering,
+ ConstrainedDivFOpLowering,
DivSIOpLowering,
DivUIOpLowering,
ExtFOpLowering,
@@ -727,6 +753,7 @@ void mlir::arith::populateArithToLLVMConversionPatterns(
MinSIOpLowering,
MinUIOpLowering,
MulFOpLowering,
+ ConstrainedMulFOpLowering,
MulIOpLowering,
MulSIExtendedOpLowering,
MulUIExtendedOpLowering,
@@ -742,6 +769,7 @@ void mlir::arith::populateArithToLLVMConversionPatterns(
ShRUIOpLowering,
SIToFPOpLowering,
SubFOpLowering,
+ ConstrainedSubFOpLowering,
SubIOpLowering,
TruncFOpLowering,
ConstrainedTruncFOpLowering,
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 76346a766f1f7..11b3aabcbfeb4 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -120,7 +120,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
if (isa<tosa::ReciprocalOp>(op) && isa<FloatType>(elementTy)) {
auto one =
arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
- return arith::DivFOp::create(rewriter, loc, resultTypes, one, args[0]);
+ return arith::DivFOp::create(rewriter, loc, one, args[0]);
}
// tosa::MulOp
@@ -140,8 +140,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
"Cannot have shift value for float");
return nullptr;
}
- return arith::MulFOp::create(rewriter, loc, resultTypes, args[0],
- args[1]);
+ return arith::MulFOp::create(rewriter, loc, args[0], args[1]);
}
if (isa<IntegerType>(elementTy)) {
@@ -538,8 +537,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
arith::ConstantOp::create(rewriter, loc, FloatAttr::get(elementTy, 1));
auto negate = arith::NegFOp::create(rewriter, loc, resultTypes, args[0]);
auto exp = mlir::math::ExpOp::create(rewriter, loc, resultTypes, negate);
- auto added = arith::AddFOp::create(rewriter, loc, resultTypes, exp, one);
- return arith::DivFOp::create(rewriter, loc, resultTypes, one, added);
+ auto added = arith::AddFOp::create(rewriter, loc, exp, one);
+ return arith::DivFOp::create(rewriter, loc, one, added);
}
// tosa::CastOp
diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index a15e19b24e54b..b822f1eadf0eb 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -437,21 +437,25 @@ def UIToFPOfExtUI :
//===----------------------------------------------------------------------===//
// mulf(negf(x), negf(y)) -> mulf(x,y)
-// (retain fastmath flags of original mulf)
+// TODO: Verify if this canonicalization is safe when a rounding mode is
+// specified. For the moment, bail on custom rounding modes.
def MulFOfNegF :
- Pat<(Arith_MulFOp (Arith_NegFOp $x, $_), (Arith_NegFOp $y, $_), $fmf),
- (Arith_MulFOp $x, $y, $fmf),
- [(Constraint<CPred<"$0.getType() == $1.getType()">> $x, $y)]>;
+ Pat<(Arith_MulFOp (Arith_NegFOp $x, $_), (Arith_NegFOp $y, $_), $fmf, $rm),
+ (Arith_MulFOp $x, $y, $fmf, $rm),
+ [(Constraint<CPred<"$0.getType() == $1.getType()">> $x, $y),
+ (Constraint<CPred<"$0 == nullptr">, "default rounding mode"> $rm)]>;
//===----------------------------------------------------------------------===//
// DivFOp
//===----------------------------------------------------------------------===//
// divf(negf(x), negf(y)) -> divf(x,y)
-// (retain fastmath flags of original divf)
+// TODO: Verify if this canonicalization is safe when a rounding mode is
+// specified. For the moment, bail on custom rounding modes.
def DivFOfNegF :
- Pat<(Arith_DivFOp (Arith_NegFOp $x, $_), (Arith_NegFOp $y, $_), $fmf),
- (Arith_DivFOp $x, $y, $fmf),
- [(Constraint<CPred<"$0.getType() == $1.getType()">> $x, $y)]>;
+ Pat<(Arith_DivFOp (Arith_NegFOp $x, $_), (Arith_NegFOp $y, $_), $fmf, $rm),
+ (Arith_DivFOp $x, $y, $fmf, $rm),
+ [(Constraint<CPred<"$0.getType() == $1.getType()">> $x, $y),
+ (Constraint<CPred<"$0 == nullptr">, "default rounding mode"> $rm)]>;
#endif // ARITH_PATTERNS
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 569d1869a5abe..e11a38ffec50c 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -34,6 +34,10 @@
using namespace mlir;
using namespace mlir::arith;
+/// Default rounding mode according to default LLVM floating-point environment.
+static constexpr llvm::RoundingMode kDefaultRoundingMode =
+ llvm::RoundingMode::NearestTiesToEven;
+
//===----------------------------------------------------------------------===//
// Pattern helpers
//===----------------------------------------------------------------------===//
@@ -105,8 +109,10 @@ arith::CmpIPredicate arith::invertPredicate(arith::CmpIPredicate pred) {
/// circular dependency with MLIRArithAttrToLLVMConversion and make arith depend
/// on the LLVM dialect and on translation to LLVM.
static llvm::RoundingMode
-convertArithRoundingModeToLLVMIR(RoundingMode roundingMode) {
- switch (roundingMode) {
+convertArithRoundingModeToLLVMIR(std::optional<RoundingMode> roundingMode) {
+ if (!roundingMode)
+ return kDefaultRoundingMode;
+ switch (*roundingMode) {
case RoundingMode::downward:
return llvm::RoundingMode::TowardNegative;
case RoundingMode::to_nearest_away:
@@ -1107,9 +1113,13 @@ OpFoldResult arith::AddFOp::fold(FoldAdaptor adaptor) {
if (matchPattern(adaptor.getRhs(), m_NegZeroFloat()))
return getLhs();
+ auto rm = getRoundingmode();
return constFoldBinaryOp<FloatAttr>(
- adaptor.getOperands(),
- [](const APFloat &a, const APFloat &b) { return a + b; });
+ adaptor.getOperands(), [rm](const APFloat &a, const APFloat &b) {
+ APFloat result(a);
+ result.add(b, convertArithRoundingModeToLLVMIR(rm));
+ return result;
+ });
}
//===----------------------------------------------------------------------===//
@@ -1121,9 +1131,13 @@ OpFoldResult arith::SubFOp::fold(FoldAdaptor adaptor) {
if (matchPattern(adaptor.getRhs(), m_PosZeroFloat()))
return getLhs();
+ auto rm = getRoundingmode();
return constFoldBinaryOp<FloatAttr>(
- adaptor.getOperands(),
- [](const APFloat &a, const APFloat &b) { return a - b; });
+ adaptor.getOperands(), [rm](const APFloat &a, const APFloat &b) {
+ APFloat result(a);
+ result.subtract(b, convertArithRoundingModeToLLVMIR(rm));
+ return result;
+ });
}
//===----------------------------------------------------------------------===//
@@ -1312,9 +1326,13 @@ OpFoldResult arith::MulFOp::fold(FoldAdaptor adaptor) {
return getRhs();
}
+ auto rm = getRoundingmode();
return constFoldBinaryOp<FloatAttr>(
- adaptor.getOperands(),
- [](const APFloat &a, const APFloat &b) { return a * b; });
+ adaptor.getOperands(), [rm](const APFloat &a, const APFloat &b) {
+ APFloat result(a);
+ result.multiply(b, convertArithRoundingModeToLLVMIR(rm));
+ return result;
+ });
}
void arith::MulFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
@@ -1331,9 +1349,13 @@ OpFoldResult arith::DivFOp::fold(FoldAdaptor adaptor) {
if (matchPattern(adaptor.getRhs(), m_OneFloat()))
return getLhs();
+ auto rm = getRoundingmode();
return constFoldBinaryOp<FloatAttr>(
- adaptor.getOperands(),
- [](const APFloat &a, const APFloat &b) { return a / b; });
+ adaptor.getOperands(), [rm](const APFloat &a, const APFloat &b) {
+ APFloat result(a);
+ result.divide(b, convertArithRoundingModeToLLVMIR(rm));
+ return result;
+ });
}
void arith::DivFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
@@ -1461,9 +1483,10 @@ static bool checkWidthChangeCast(TypeRange inputs, TypeRange outputs) {
/// Attempts to convert `sourceValue` to an APFloat value with
/// `targetSemantics` and `roundingMode`, without any information loss.
-static FailureOr<APFloat> convertFloatValue(
- APFloat sourceValue, const llvm::fltSemantics &targetSemantics,
- llvm::RoundingMode roundingMode = llvm::RoundingMode::NearestTiesToEven) {
+static FailureOr<APFloat>
+convertFloatValue(APFloat sourceValue,
+ const llvm::fltSemantics &targetSemantics,
+ llvm::RoundingMode roundingMode = kDefaultRoundingMode) {
// Reject special values that are not representable in the target type before
// calling APFloat::convert, which would llvm_unreachable on them.
using fltNonfiniteBehavior = llvm::fltNonfiniteBehavior;
@@ -1685,10 +1708,8 @@ OpFoldResult arith::TruncFOp::fold(FoldAdaptor adaptor) {
return constFoldCastOp<FloatAttr, FloatAttr>(
adaptor.getOperands(), getType(),
[this, &targetSemantics](const APFloat &a, bool &castStatus) {
- RoundingMode roundingMode =
- getRoundingmode().value_or(RoundingMode::to_nearest_even);
llvm::RoundingMode llvmRoundingMode =
- convertArithRoundingModeToLLVMIR(roundingMode);
+ convertArithRoundingModeToLLVMIR(getRoundingmode());
FailureOr<APFloat> result =
convertFloatValue(a, targetSemantics, llvmRoundingMode);
if (failed(result)) {
@@ -1722,10 +1743,8 @@ OpFoldResult arith::ConvertFOp::fold(FoldAdaptor adaptor) {
return constFoldCastOp<FloatAttr, FloatAttr>(
adaptor.getOperands(), getType(),
[this, &targetSemantics](const APFloat &a, bool &castStatus) {
- RoundingMode roundingMode =
- getRoundingmode().value_or(RoundingMode::to_nearest_even);
llvm::RoundingMode llvmRoundingMode =
- convertArithRoundingModeToLLVMIR(roundingMode);
+ convertArithRoundingModeToLLVMIR(getRoundingmode());
FailureOr<APFloat> result =
convertFloatValue(a, targetSemantics, llvmRoundingMode);
if (failed(result)) {
diff --git a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
index 75601e215744c..df58d4ffcaf51 100644
--- a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
+++ b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
@@ -401,6 +401,68 @@ func.func @experimental_constrained_fptrunc(%arg0 : f64) {
// -----
+// CHECK-LABEL: experimental_constrained_addf
+func.func @experimental_constrained_addf(%arg0 : f64, %arg1 : f64) {
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fadd %arg0, %arg1 tonearest ignore
+ %0 = arith.addf %arg0, %arg1 to_nearest_even : f64
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fadd %arg0, %arg1 downward ignore
+ %1 = arith.addf %arg0, %arg1 downward : f64
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fadd %arg0, %arg1 upward ignore
+ %2 = arith.addf %arg0, %arg1 upward : f64
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fadd %arg0, %arg1 towardzero ignore
+ %3 = arith.addf %arg0, %arg1 toward_zero : f64
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fadd %arg0, %arg1 tonearestaway ignore
+ %4 = arith.addf %arg0, %arg1 to_nearest_away : f64
+ return
+}
+
+// -----
+
+// CHECK-LABEL: experimental_constrained_subf
+func.func @experimental_constrained_subf(%arg0 : f64, %arg1 : f64) {
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fsub %arg0, %arg1 tonearest ignore
+ %0 = arith.subf %arg0, %arg1 to_nearest_even : f64
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fsub %arg0, %arg1 downward ignore
+ %1 = arith.subf %arg0, %arg1 downward : f64
+ return
+}
+
+// -----
+
+// CHECK-LABEL: experimental_constrained_mulf
+func.func @experimental_constrained_mulf(%arg0 : f64, %arg1 : f64) {
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fmul %arg0, %arg1 upward ignore
+ %0 = arith.mulf %arg0, %arg1 upward : f64
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fmul %arg0, %arg1 towardzero ignore
+ %1 = arith.mulf %arg0, %arg1 toward_zero : f64
+ return
+}
+
+// -----
+
+// CHECK-LABEL: experimental_constrained_divf
+func.func @experimental_constrained_divf(%arg0 : f64, %arg1 : f64) {
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fdiv %arg0, %arg1 tonearest ignore
+ %0 = arith.divf %arg0, %arg1 to_nearest_even : f64
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fdiv %arg0, %arg1 tonearestaway ignore
+ %1 = arith.divf %arg0, %arg1 to_nearest_away : f64
+ return
+}
+
+// -----
+
+// Verify that fastmath flags are stripped when lowering to constrained
+// intrinsics (constrained FP and fastmath are contradictory).
+// CHECK-LABEL: constrained_addf_with_fastmath
+func.func @constrained_addf_with_fastmath(%arg0 : f64, %arg1 : f64) {
+// CHECK-NEXT: = llvm.intr.experimental.constrained.fadd %arg0, %arg1 tonearest ignore : f64
+// CHECK-NOT: fastmath
+ %0 = arith.addf %arg0, %arg1 to_nearest_even fastmath<fast> : f64
+ return
+}
+
+// -----
+
// CHECK-LABEL: @convertf_f16_to_bf16
func.func @convertf_f16_to_bf16(%arg0 : f16) -> bf16 {
// CHECK-NEXT: %[[EXT:.*]] = llvm.fpext %arg0 : f16 to f32
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index ee3e713f8481e..b153bd7c32261 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -2554,6 +2554,81 @@ func.func @test_divf1(%arg0 : f32, %arg1 : f32) -> (f32) {
// -----
+// Verify that constant folding respects rounding modes. 1.0000001 + 1.0 is not
+// exactly representable in f32. With upward rounding, the result is rounded up,
+// and with downward rounding it is rounded down.
+// CHECK-LABEL: @test_addf_rounding_mode(
+// CHECK-SAME: %[[ARG0:.+]]: f32
+func.func @test_addf_rounding_mode(%arg0 : f32) -> (f32, f32, f32) {
+ // CHECK-DAG: %[[UP:.+]] = arith.constant 2.00000024 : f32
+ // CHECK-DAG: %[[DOWN:.+]] = arith.constant 2.000000e+00 : f32
+ // CHECK-NEXT: return %[[ARG0]], %[[UP]], %[[DOWN]]
+ %a = arith.constant 1.0000001 : f32
+ %b = arith.constant 1.0 : f32
+ // addf(x, -0) folds even with a rounding mode.
+ %c_neg0 = arith.constant -0.0 : f32
+ %0 = arith.addf %arg0, %c_neg0 to_nearest_even : f32
+ %1 = arith.addf %a, %b upward : f32
+ %2 = arith.addf %a, %b downward : f32
+ return %0, %1, %2 : f32, f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: @test_subf_rounding_mode(
+// CHECK-SAME: %[[ARG0:.+]]: f32
+func.func @test_subf_rounding_mode(%arg0 : f32) -> (f32, f32, f32) {
+ // CHECK-DAG: %[[UP:.+]] = arith.constant 2.00000024 : f32
+ // CHECK-DAG: %[[DOWN:.+]] = arith.constant 2.000000e+00 : f32
+ // CHECK-NEXT: return %[[ARG0]], %[[UP]], %[[DOWN]]
+ %a = arith.constant 1.0000001 : f32
+ %b = arith.constant -1.0 : f32
+ // subf(x, +0) folds even with a rounding mode.
+ %c0 = arith.constant 0.0 : f32
+ %0 = arith.subf %arg0, %c0 downward : f32
+ %1 = arith.subf %a, %b upward : f32
+ %2 = arith.subf %a, %b downward : f32
+ return %0, %1, %2 : f32, f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: @test_mulf_rounding_mode(
+// CHECK-SAME: %[[ARG0:.+]]: f32
+func.func @test_mulf_rounding_mode(%arg0 : f32) -> (f32, f32, f32) {
+ // CHECK-DAG: %[[UP:.+]] = arith.constant 3.00000048 : f32
+ // CHECK-DAG: %[[DOWN:.+]] = arith.constant 3.00000024 : f32
+ // CHECK-NEXT: return %[[ARG0]], %[[UP]], %[[DOWN]]
+ %a = arith.constant 1.0000001 : f32
+ %b = arith.constant 3.0 : f32
+ // mulf(x, 1) folds even with a rounding mode.
+ %c1 = arith.constant 1.0 : f32
+ %0 = arith.mulf %arg0, %c1 upward : f32
+ %1 = arith.mulf %a, %b upward : f32
+ %2 = arith.mulf %a, %b downward : f32
+ return %0, %1, %2 : f32, f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: @test_divf_rounding_mode(
+// CHECK-SAME: %[[ARG0:.+]]: f32
+func.func @test_divf_rounding_mode(%arg0 : f32) -> (f32, f32, f32) {
+ // CHECK-DAG: %[[UP:.+]] = arith.constant 0.333333343 : f32
+ // CHECK-DAG: %[[DOWN:.+]] = arith.constant 0.333333313 : f32
+ // CHECK-NEXT: return %[[ARG0]], %[[UP]], %[[DOWN]]
+ %a = arith.constant 1.0 : f32
+ %b = arith.constant 3.0 : f32
+ // divf(x, 1) folds even with a rounding mode.
+ %c1 = arith.constant 1.0 : f32
+ %0 = arith.divf %arg0, %c1 toward_zero : f32
+ %1 = arith.divf %a, %b upward : f32
+ %2 = arith.divf %a, %b downward : f32
+ return %0, %1, %2 : f32, f32, f32
+}
+
+// -----
+
func.func @fold_divui_of_muli_0(%arg0 : index, %arg1 : index) -> index {
%0 = arith.muli %arg0, %arg1 overflow<nuw> : index
%1 = arith.divui %0, %arg0 : index
diff --git a/mlir/test/Dialect/Arith/ops.mlir b/mlir/test/Dialect/Arith/ops.mlir
index 2c5371de9ff24..3874c85818eb4 100644
--- a/mlir/test/Dialect/Arith/ops.mlir
+++ b/mlir/test/Dialect/Arith/ops.mlir
@@ -1234,6 +1234,21 @@ func.func @fastmath(%arg0: f32, %arg1: f32, %arg2: i32) {
return
}
+// CHECK-LABEL: @roundingmode
+func.func @roundingmode(%arg0: f32, %arg1: f32) {
+// CHECK: {{.*}} = arith.addf %arg0, %arg1 to_nearest_even : f32
+ %0 = arith.addf %arg0, %arg1 to_nearest_even : f32
+// CHECK: {{.*}} = arith.subf %arg0, %arg1 downward : f32
+ %1 = arith.subf %arg0, %arg1 downward : f32
+// CHECK: {{.*}} = arith.mulf %arg0, %arg1 upward : f32
+ %2 = arith.mulf %arg0, %arg1 upward : f32
+// CHECK: {{.*}} = arith.divf %arg0, %arg1 toward_zero : f32
+ %3 = arith.divf %arg0, %arg1 toward_zero : f32
+// CHECK: {{.*}} = arith.addf %arg0, %arg1 to_nearest_even fastmath<fast> : f32
+ %4 = arith.addf %arg0, %arg1 to_nearest_even fastmath<fast> : f32
+ return
+}
+
// CHECK-LABEL: @select_tensor
func.func @select_tensor(%arg0 : tensor<8xi1>, %arg1 : tensor<8xi32>, %arg2 : tensor<8xi32>) -> tensor<8xi32> {
// CHECK: = arith.select %{{.*}}, %{{.*}}, %{{.*}} : tensor<8xi1>, tensor<8xi32>
More information about the Mlir-commits
mailing list