[clang] [CIR] Add fast-math flags to cir.fma, cir.fmuladd and cir.fneg (PR #237732)
Arseniy Obolenskiy via cfe-commits
cfe-commits at lists.llvm.org
Sun Nov 1 05:43:31 PST 2026
https://github.com/aobolensk created https://github.com/llvm/llvm-project/pull/237732
Classic codegen keeps fast-math flags on these ops, but CIR dropped them
Carry the flags in an optional attribute and forward it to the LLVM dialect
>From 8d375464fd8a540dd09817712714418f6cf32bc5 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Sun, 1 Nov 2026 14:42:52 +0100
Subject: [PATCH] [CIR] Add fast-math flags to cir.fma, cir.fmuladd and
cir.fneg
Classic codegen keeps fast-math flags on these ops, but CIR dropped them
Carry the flags in an optional attribute and forward it to the LLVM dialect
---
.../CIR/Dialect/Builder/CIRBaseBuilder.h | 3 +-
clang/include/clang/CIR/Dialect/IR/CIROps.td | 26 +++++++++++++--
clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 3 +-
.../lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp | 3 +-
clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 13 +++++---
.../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 33 ++++++++++++-------
.../CIR/Lowering/DirectToLLVM/LowerToLLVM.h | 6 ++++
clang/test/CIR/CodeGen/fast-math-flags.c | 16 +++++++++
clang/test/CIR/CodeGen/fp-contract.c | 22 +++++++++++++
clang/utils/TableGen/CIRLoweringEmitter.cpp | 11 +++++++
10 files changed, 113 insertions(+), 23 deletions(-)
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index 489ed57695d299..cd1e950a665399 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -947,10 +947,9 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
assert(cir::isFPOrVectorOfFPType(operand.getType()) &&
"expected floating-point or vector-of-float type");
assert(!cir::MissingFeatures::metaDataNode());
- assert(!cir::MissingFeatures::fastMathFlags());
// fneg does not raise FP exceptions or depend on the rounding mode, so it
// never carries an fenv attribute.
- return cir::FNegOp::create(*this, loc, operand);
+ return cir::FNegOp::create(*this, loc, operand, getFastMathFlagsAttr());
}
mlir::Value createXor(mlir::Location loc, mlir::Value lhs, mlir::Value rhs) {
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index bd762aae3e1d7e..990a84da651de4 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -2196,6 +2196,9 @@ def CIR_FNegOp : CIR_UnaryOp<"fneg", CIR_AnyFloatOrVecOfFloatType> {
The `cir.fneg` operation negates the operand. The operand and result must
have the same type.
+ The optional `fastmath_flags` attribute holds the fast-math flags of the
+ operation.
+
Example:
```
@@ -2205,6 +2208,19 @@ def CIR_FNegOp : CIR_UnaryOp<"fneg", CIR_AnyFloatOrVecOfFloatType> {
```
}];
+ let arguments = !con(commonArgs, (ins
+ OptionalAttr<CIR_FastMathFlagsAttr>:$fastmath_flags));
+
+ let assemblyFormat = [{
+ $input `:` type($input) prop-dict attr-dict
+ }];
+
+ let builders = [
+ OpBuilder<(ins "mlir::Value":$input), [{
+ build($_builder, $_state, input, cir::FastMathFlagsAttr{});
+ }]>
+ ];
+
let hasFolder = 1;
let llvmOp = "FNegOp";
@@ -8802,6 +8818,8 @@ def CIR_ModfOp : CIR_Op<"modf", [Pure]> {
}];
}
+// The optional `fastmath_flags` attribute holds the fast-math flags of the
+// operation.
class CIR_TernaryFPToFPBuiltinOp<string mnemonic, string llvmOpName>
: CIR_Op<mnemonic,
!listconcat([SameOperandsAndResultType], CIR_FenvOpTraits)>
@@ -8810,19 +8828,21 @@ class CIR_TernaryFPToFPBuiltinOp<string mnemonic, string llvmOpName>
CIR_AnyFloatOrVecOfFloatType:$a,
CIR_AnyFloatOrVecOfFloatType:$b,
CIR_AnyFloatOrVecOfFloatType:$c,
- OptionalAttr<CIR_FenvAttr>:$fenv
+ OptionalAttr<CIR_FenvAttr>:$fenv,
+ OptionalAttr<CIR_FastMathFlagsAttr>:$fastmath_flags
);
let results = (outs CIR_AnyFloatOrVecOfFloatType:$result);
let assemblyFormat = [{
- $a `,` $b `,` $c `:` type($a) (`fenv` `` $fenv^)? attr-dict
+ $a `,` $b `,` $c `:` type($a) (`fenv` `` $fenv^)? prop-dict attr-dict
}];
let builders = [
OpBuilder<(ins "mlir::Type":$result, "mlir::Value":$a, "mlir::Value":$b,
"mlir::Value":$c), [{
- build($_builder, $_state, result, a, b, c, cir::FenvAttr{});
+ build($_builder, $_state, result, a, b, c, cir::FenvAttr{},
+ cir::FastMathFlagsAttr{});
}]>
];
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 55941a8480cf1d..e1b50240da8ff2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -632,7 +632,8 @@ static RValue emitTernaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
mlir::Type ty = cgf.convertType(e.getType());
auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1, arg2,
- cgf.getBuilder().getConstrainedFPAttr());
+ cgf.getBuilder().getConstrainedFPAttr(),
+ cgf.getBuilder().getFastMathFlagsAttr());
return RValue::get(call->getResult(0));
}
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
index 9f9b34f80ec3ef..591625def330ed 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
@@ -172,7 +172,8 @@ emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder,
if constexpr (std::is_same_v<Operation, cir::FMAOp>) {
assert(args.size() == 3 && "fma expects three operands");
return Operation::create(builder, loc, funcResTy, args[0], args[1], args[2],
- builder.getConstrainedFPAttr())
+ builder.getConstrainedFPAttr(),
+ builder.getFastMathFlagsAttr())
.getResult();
}
if constexpr (std::is_same_v<Operation, cir::SqrtOp>) {
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 8e6ded0527cc90..0801ed6c4ab71a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -887,8 +887,11 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
mlir::Location loc = cgf.getLoc(e->getSourceRange().getBegin());
- if (cir::isFPOrVectorOfFPType(operand.getType()))
- return builder.createOrFold<cir::FNegOp>(loc, operand);
+ if (cir::isFPOrVectorOfFPType(operand.getType())) {
+ CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
+ return builder.createOrFold<cir::FNegOp>(loc, operand,
+ builder.getFastMathFlagsAttr());
+ }
// TODO(cir): We might have to change this to support overflow trapping.
// Classic codegen routes unary minus through emitSub to ensure
@@ -2110,9 +2113,9 @@ static mlir::Value buildFMulAdd(mlir::Location addLoc, cir::FMulOp mulOp,
// Carry the mul's fenv attribute so a constrained fmul yields a constrained
// fmuladd; the builder is under the add's FP options, not the mul's.
- mlir::Value fmuladd =
- cir::FMulAddOp::create(builder, loc, addend.getType(), mulOp0, mulOp1,
- addend, mulOp.getFenvAttr());
+ mlir::Value fmuladd = cir::FMulAddOp::create(
+ builder, loc, addend.getType(), mulOp0, mulOp1, addend,
+ mulOp.getFenvAttr(), builder.getFastMathFlagsAttr());
mulOp.erase();
return fmuladd;
}
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 9d74696bde0d39..8338ad1ea9ac4b 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -680,12 +680,34 @@ static mlir::Value createConstrainedFPIntrinsicCall(
return intrinsic->getResult(0);
}
+template <typename LLVMOp>
+mlir::LogicalResult lowerFPOp(mlir::Operation *op, mlir::ValueRange operands,
+ cir::FastMathFlagsAttr fastmath,
+ const mlir::TypeConverter &typeConverter,
+ mlir::ConversionPatternRewriter &rewriter) {
+ mlir::Type llvmResTy = typeConverter.convertType(op->getResultTypes()[0]);
+ if (!llvmResTy)
+ return op->emitError("expected LLVM result type for floating-point op");
+
+ auto newOp = rewriter.replaceOpWithNewOp<LLVMOp>(
+ op, mlir::TypeRange{llvmResTy}, operands,
+ cir::getDefaultProperties<LLVMOp>(op->getContext()));
+ if (fastmath)
+ mlir::cast<mlir::LLVM::FastmathFlagsInterface>(newOp.getOperation())
+ .setFastmathAttr(mlir::LLVM::FastmathFlagsAttr::get(
+ rewriter.getContext(), convertFastMathFlags(fastmath.getValue())));
+ return mlir::success();
+}
+
template <typename LLVMOp>
mlir::LogicalResult lowerConstrainableFPOp(
mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv,
cir::FastMathFlagsAttr fastmath, const mlir::TypeConverter &typeConverter,
mlir::ConversionPatternRewriter &rewriter,
llvm::StringRef constrainedMnemonic, bool hasRoundingMode) {
+ if (!fenv)
+ return lowerFPOp<LLVMOp>(op, operands, fastmath, typeConverter, rewriter);
+
mlir::Type llvmResTy = typeConverter.convertType(op->getResultTypes()[0]);
if (!llvmResTy)
return op->emitError("expected LLVM result type for floating-point op");
@@ -694,17 +716,6 @@ mlir::LogicalResult lowerConstrainableFPOp(
if (fastmath)
fastmathFlags = convertFastMathFlags(fastmath.getValue());
- if (!fenv) {
- auto newOp = rewriter.replaceOpWithNewOp<LLVMOp>(
- op, mlir::TypeRange{llvmResTy}, operands,
- cir::getDefaultProperties<LLVMOp>(op->getContext()));
- if (fastmathFlags != mlir::LLVM::FastmathFlags::none)
- mlir::cast<mlir::LLVM::FastmathFlagsInterface>(newOp.getOperation())
- .setFastmathAttr(mlir::LLVM::FastmathFlagsAttr::get(
- rewriter.getContext(), fastmathFlags));
- return mlir::success();
- }
-
return lowerToConstrainedFPIntrinsic(op, operands, fenv, llvmResTy, rewriter,
constrainedMnemonic, hasRoundingMode,
fastmathFlags);
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index d3ef1d968f3856..426e71f8a5561e 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -93,6 +93,12 @@ mlir::LogicalResult lowerToConstrainedFPIntrinsic(
llvm::StringRef constrainedMnemonic, bool hasRoundingMode,
mlir::LLVM::FastmathFlags fastmathFlags = {});
+template <typename LLVMOp>
+mlir::LogicalResult lowerFPOp(mlir::Operation *op, mlir::ValueRange operands,
+ cir::FastMathFlagsAttr fastmath,
+ const mlir::TypeConverter &typeConverter,
+ mlir::ConversionPatternRewriter &rewriter);
+
template <typename LLVMOp>
mlir::LogicalResult lowerConstrainableFPOp(
mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv,
diff --git a/clang/test/CIR/CodeGen/fast-math-flags.c b/clang/test/CIR/CodeGen/fast-math-flags.c
index cd2e74b4e5ab42..1cf3dc53fb00bb 100644
--- a/clang/test/CIR/CodeGen/fast-math-flags.c
+++ b/clang/test/CIR/CodeGen/fast-math-flags.c
@@ -224,3 +224,19 @@ float4 vector(float4 a, float4 b, float4 c) {
// LLVM-FAST: fsub fast <4 x float> %{{.*}}, %{{.*}}
// LLVM-FAST: fmul fast <4 x float> %{{.*}}, %{{.*}}
// LLVM-FAST: fdiv fast <4 x float> %{{.*}}, %{{.*}}
+
+float fma_builtin(float a, float b, float c) {
+ return __builtin_fmaf(a, b, c);
+}
+// CIR-LABEL: cir.func {{.*}}@fma_builtin(
+// LLVM-LABEL: define {{.*}}@fma_builtin(
+// CIR-FAST-NO-CONTRACT: cir.fma %{{.*}}, %{{.*}}, %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn, reassoc]>{{( loc.*)?$}}
+// LLVM-FAST-NO-CONTRACT: call reassoc nnan ninf nsz arcp afn float @llvm.fma.f32(
+
+float unary_minus(float a) {
+ return -a;
+}
+// CIR-LABEL: cir.func {{.*}}@unary_minus(
+// LLVM-LABEL: define {{.*}}@unary_minus(
+// CIR-FAST-NO-CONTRACT: cir.fneg %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn, reassoc]>{{( loc.*)?$}}
+// LLVM-FAST-NO-CONTRACT: fneg reassoc nnan ninf nsz arcp afn float %{{.*}}
diff --git a/clang/test/CIR/CodeGen/fp-contract.c b/clang/test/CIR/CodeGen/fp-contract.c
index 746d6e991f691a..0ec886e686652b 100644
--- a/clang/test/CIR/CodeGen/fp-contract.c
+++ b/clang/test/CIR/CodeGen/fp-contract.c
@@ -51,6 +51,14 @@
// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -ffp-contract=fast -fexperimental-strict-floating-point -ffp-exception-behavior=strict -emit-llvm %s -o %t-strict-fast-ogcg.ll
// RUN: FileCheck --input-file=%t-strict-fast-ogcg.ll %s -check-prefix=LLVM-STRICT-FAST
+// The fused op carries the remaining fast-math flags.
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -ffp-contract=on -menable-no-infs -menable-no-nans -fno-signed-zeros -freciprocal-math -fapprox-func -emit-cir %s -o %t-fmf.cir
+// RUN: FileCheck --input-file=%t-fmf.cir %s -check-prefix=CIR-FMF
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -ffp-contract=on -menable-no-infs -menable-no-nans -fno-signed-zeros -freciprocal-math -fapprox-func -emit-llvm %s -o %t-fmf.ll
+// RUN: FileCheck --input-file=%t-fmf.ll %s -check-prefix=LLVM-FMF
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -ffp-contract=on -menable-no-infs -menable-no-nans -fno-signed-zeros -freciprocal-math -fapprox-func -emit-llvm %s -o %t-fmf-ogcg.ll
+// RUN: FileCheck --input-file=%t-fmf-ogcg.ll %s -check-prefix=LLVM-FMF
+
// a * b + c => fmuladd(a, b, c)
float fmuladd_add(float a, float b, float c) {
return a * b + c;
@@ -79,6 +87,12 @@ float fmuladd_add(float a, float b, float c) {
// LLVM-FAST: fmul contract float
// LLVM-FAST: fadd contract float
+// CIR-FMF-LABEL: cir.func {{.*}}@fmuladd_add
+// CIR-FMF: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn]>
+
+// LLVM-FMF-LABEL: @fmuladd_add
+// LLVM-FMF: call nnan ninf nsz arcp afn float @llvm.fmuladd.f32
+
// c + a * b => fmuladd(a, b, c) (mul on the RHS)
float fmuladd_add_rhs(float a, float b, float c) {
@@ -102,6 +116,14 @@ float fmuladd_sub(float a, float b, float c) {
// LLVM-ON: %[[NEG:.*]] = fneg float
// LLVM-ON: call float @llvm.fmuladd.f32(float %{{.*}}, float %{{.*}}, float %[[NEG]])
+// CIR-FMF-LABEL: cir.func {{.*}}@fmuladd_sub
+// CIR-FMF: cir.fneg %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn]>
+// CIR-FMF: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn]>
+
+// LLVM-FMF-LABEL: @fmuladd_sub
+// LLVM-FMF: fneg nnan ninf nsz arcp afn float
+// LLVM-FMF: call nnan ninf nsz arcp afn float @llvm.fmuladd.f32
+
// If the mul result is used elsewhere, it must NOT be fused.
float no_fmuladd_reused_mul(float a, float b, float c, float *p) {
float m = a * b;
diff --git a/clang/utils/TableGen/CIRLoweringEmitter.cpp b/clang/utils/TableGen/CIRLoweringEmitter.cpp
index 46a8dd8bd6ba0d..44fe1143938af5 100644
--- a/clang/utils/TableGen/CIRLoweringEmitter.cpp
+++ b/clang/utils/TableGen/CIRLoweringEmitter.cpp
@@ -223,6 +223,17 @@ void GenerateLLVMLoweringPattern(llvm::StringRef OpName,
Code << " /*hasRoundingMode=*/"
<< (ConstrainedHasRoundingMode ? "true" : "false") << ");\n";
Code << " }\n";
+ } else if (!LLVMOp.empty() && HasFastMathFlags) {
+ assert(!HasZeroResult && "fast-math op must have a result");
+ Code
+ << " mlir::LogicalResult matchAndRewrite(cir::" << OpName
+ << " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "
+ "const override {\n";
+ Code << " return lowerFPOp<mlir::LLVM::" << LLVMOp
+ << ">(op, adaptor.getOperands(),\n";
+ Code << " op.getFastmathFlagsAttr(), *getTypeConverter(), "
+ "rewriter);\n";
+ Code << " }\n";
} else if (!LLVMOp.empty()) {
// Generate the matchAndRewrite body automatically.
Code
More information about the cfe-commits
mailing list