[Mlir-commits] [mlir] [mlir][Vector] Add fastmath flags to vector.reduction (PR #66905)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Wed Sep 20 06:43:25 PDT 2023
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-mlir-vector
<details>
<summary>Changes</summary>
This revision pipes the fastmath attribute support through the vector.reduction op. This seemingly simple first step already requires quite some genuflexions, file and builder reorganization. In the process, retire the boolean reassoc flag deep in the LLVM dialect builders and just use the fastmath attribute.
During conversions, templated builders for predicated intrinsics are partially cleaned up. In the future, to finalize the cleanups, one should consider adding fastmath to the VPIntrinsic ops.
---
Patch is 36.78 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/66905.diff
10 Files Affected:
- (modified) mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td (+4-9)
- (modified) mlir/include/mlir/Dialect/Vector/IR/CMakeLists.txt (+16-8)
- (added) mlir/include/mlir/Dialect/Vector/IR/Vector.td (+31)
- (added) mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td (+85)
- (modified) mlir/include/mlir/Dialect/Vector/IR/VectorOps.h (+4-3)
- (modified) mlir/include/mlir/Dialect/Vector/IR/VectorOps.td (+20-87)
- (modified) mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp (+78-78)
- (modified) mlir/lib/Dialect/Vector/IR/CMakeLists.txt (+1-1)
- (modified) mlir/lib/Dialect/Vector/IR/VectorOps.cpp (+20-8)
- (modified) mlir/python/mlir/dialects/VectorOps.td (+5-5)
``````````diff
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 51017b5e050ffef..5af84c9e8646f5a 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -656,8 +656,9 @@ class LLVM_VecReductionI<string mnem>
class LLVM_VecReductionAccBase<string mnem, Type element>
: LLVM_OneResultIntrOp<"vector.reduce." # mnem, [], [0],
[Pure, SameOperandsAndResultElementType]>,
- Arguments<(ins element:$start_value, LLVM_VectorOf<element>:$input,
- DefaultValuedAttr<BoolAttr, "false">:$reassoc)> {
+ Arguments<(ins element:$start_value,
+ LLVM_VectorOf<element>:$input,
+ DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags)> {
let llvmBuilder = [{
llvm::Module *module = builder.GetInsertBlock()->getModule();
llvm::Function *fn = llvm::Intrinsic::getDeclaration(
@@ -667,17 +668,11 @@ class LLVM_VecReductionAccBase<string mnem, Type element>
", ") # [{
});
auto operands = moduleTranslation.lookupValues(opInst.getOperands());
- llvm::FastMathFlags origFM = builder.getFastMathFlags();
- llvm::FastMathFlags tempFM = origFM;
- tempFM.setAllowReassoc($reassoc);
- builder.setFastMathFlags(tempFM); // set fastmath flag
$res = builder.CreateCall(fn, operands);
- builder.setFastMathFlags(origFM); // restore fastmath flag
}];
let mlirBuilder = [{
- bool allowReassoc = inst->getFastMathFlags().allowReassoc();
$res = $_builder.create<$_qualCppClassName>($_location,
- $_resultType, $start_value, $input, allowReassoc);
+ $_resultType, $start_value, $input, inst->getFastMathFlags());
}];
}
diff --git a/mlir/include/mlir/Dialect/Vector/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Vector/IR/CMakeLists.txt
index 2e56afe727ac0c9..23bed7e0f447e42 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Vector/IR/CMakeLists.txt
@@ -1,10 +1,18 @@
-add_mlir_dialect(VectorOps vector)
-add_mlir_doc(VectorOps VectorOps Dialects/ -gen-op-doc)
+add_mlir_dialect(Vector vector)
+add_mlir_doc(Vector Vector Dialects/ -gen-op-doc -dialect=vector)
+# Add Vector operations
set(LLVM_TARGET_DEFINITIONS VectorOps.td)
-mlir_tablegen(VectorOpsEnums.h.inc -gen-enum-decls)
-mlir_tablegen(VectorOpsEnums.cpp.inc -gen-enum-defs)
-mlir_tablegen(VectorOpsAttrDefs.h.inc -gen-attrdef-decls)
-mlir_tablegen(VectorOpsAttrDefs.cpp.inc -gen-attrdef-defs)
-add_public_tablegen_target(MLIRVectorOpsEnumsIncGen)
-add_dependencies(mlir-headers MLIRVectorOpsEnumsIncGen)
+mlir_tablegen(VectorOps.h.inc -gen-op-decls)
+mlir_tablegen(VectorOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRVectorOpsIncGen)
+add_dependencies(mlir-generic-headers MLIRVectorOpsIncGen)
+
+# Add Vector attributes
+set(LLVM_TARGET_DEFINITIONS VectorAttributes.td)
+mlir_tablegen(VectorEnums.h.inc -gen-enum-decls)
+mlir_tablegen(VectorEnums.cpp.inc -gen-enum-defs)
+mlir_tablegen(VectorAttributes.h.inc -gen-attrdef-decls)
+mlir_tablegen(VectorAttributes.cpp.inc -gen-attrdef-defs)
+add_public_tablegen_target(MLIRVectorAttributesIncGen)
+add_dependencies(mlir-generic-headers MLIRVectorAttributesIncGen)
diff --git a/mlir/include/mlir/Dialect/Vector/IR/Vector.td b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
new file mode 100644
index 000000000000000..c439ca083e2e09b
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
@@ -0,0 +1,31 @@
+//===- Vector.td - Vector Dialect --------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the Vector dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_VECTOR_IR_VECTOR
+#define MLIR_DIALECT_VECTOR_IR_VECTOR
+
+include "mlir/IR/OpBase.td"
+
+def Vector_Dialect : Dialect {
+ let name = "vector";
+ let cppNamespace = "::mlir::vector";
+
+ let useDefaultAttributePrinterParser = 1;
+ let hasConstantMaterializer = 1;
+ let dependentDialects = ["arith::ArithDialect"];
+}
+
+// Base class for Vector dialect ops.
+class Vector_Op<string mnemonic, list<Trait> traits = []> :
+ Op<Vector_Dialect, mnemonic, traits>;
+
+#endif // MLIR_DIALECT_VECTOR_IR_VECTOR
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td b/mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td
new file mode 100644
index 000000000000000..2db944b4ceaf139
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td
@@ -0,0 +1,85 @@
+//===- VectorAttributes.td - Vector Dialect ----------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the attributes used in the Vector dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_VECTOR_IR_VECTOR_ATTRIBUTES
+#define MLIR_DIALECT_VECTOR_IR_VECTOR_ATTRIBUTES
+
+include "Vector.td"
+include "mlir/IR/EnumAttr.td"
+
+// The "kind" of combining function for contractions and reductions.
+def COMBINING_KIND_ADD : I32BitEnumAttrCaseBit<"ADD", 0, "add">;
+def COMBINING_KIND_MUL : I32BitEnumAttrCaseBit<"MUL", 1, "mul">;
+def COMBINING_KIND_MINUI : I32BitEnumAttrCaseBit<"MINUI", 2, "minui">;
+def COMBINING_KIND_MINSI : I32BitEnumAttrCaseBit<"MINSI", 3, "minsi">;
+def COMBINING_KIND_MINF : I32BitEnumAttrCaseBit<"MINF", 4, "minf">;
+def COMBINING_KIND_MAXUI : I32BitEnumAttrCaseBit<"MAXUI", 5, "maxui">;
+def COMBINING_KIND_MAXSI : I32BitEnumAttrCaseBit<"MAXSI", 6, "maxsi">;
+def COMBINING_KIND_MAXF : I32BitEnumAttrCaseBit<"MAXF", 7, "maxf">;
+def COMBINING_KIND_AND : I32BitEnumAttrCaseBit<"AND", 8, "and">;
+def COMBINING_KIND_OR : I32BitEnumAttrCaseBit<"OR", 9, "or">;
+def COMBINING_KIND_XOR : I32BitEnumAttrCaseBit<"XOR", 10, "xor">;
+def COMBINING_KIND_MINIMUMF : I32BitEnumAttrCaseBit<"MINIMUMF", 11, "minimumf">;
+def COMBINING_KIND_MAXIMUMF : I32BitEnumAttrCaseBit<"MAXIMUMF", 12, "maximumf">;
+
+def CombiningKind : I32BitEnumAttr<
+ "CombiningKind",
+ "Kind of combining function for contractions and reductions",
+ [COMBINING_KIND_ADD, COMBINING_KIND_MUL, COMBINING_KIND_MINUI,
+ COMBINING_KIND_MINSI, COMBINING_KIND_MINF, COMBINING_KIND_MAXUI,
+ COMBINING_KIND_MAXSI, COMBINING_KIND_MAXF, COMBINING_KIND_AND,
+ COMBINING_KIND_OR, COMBINING_KIND_XOR,
+ COMBINING_KIND_MAXIMUMF, COMBINING_KIND_MINIMUMF]> {
+ let cppNamespace = "::mlir::vector";
+ let genSpecializedAttr = 0;
+}
+
+/// An attribute that specifies the combining function for `vector.contract`,
+/// and `vector.reduction`.
+def Vector_CombiningKindAttr : EnumAttr<Vector_Dialect, CombiningKind, "kind"> {
+ let assemblyFormat = "`<` $value `>`";
+}
+
+def Vector_IteratorType : I32EnumAttr<"IteratorType", "Iterator type", [
+ I32EnumAttrCase<"parallel", 0>,
+ I32EnumAttrCase<"reduction", 1>
+]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::vector";
+}
+
+def Vector_IteratorTypeEnum
+ : EnumAttr<Vector_Dialect, Vector_IteratorType, "iterator_type"> {
+ let assemblyFormat = "`<` $value `>`";
+}
+
+def Vector_IteratorTypeArrayAttr
+ : TypedArrayAttrBase<Vector_IteratorTypeEnum,
+ "Iterator type should be an enum.">;
+
+def PrintPunctuation : I32EnumAttr<"PrintPunctuation",
+ "Punctuation for separating vectors or vector elements", [
+ I32EnumAttrCase<"NoPunctuation", 0, "no_punctuation">,
+ I32EnumAttrCase<"NewLine", 1, "newline">,
+ I32EnumAttrCase<"Comma", 2, "comma">,
+ I32EnumAttrCase<"Open", 3, "open">,
+ I32EnumAttrCase<"Close", 4, "close">
+]> {
+ let cppNamespace = "::mlir::vector";
+ let genSpecializedAttr = 0;
+}
+
+def Vector_PrintPunctuation : EnumAttr<Vector_Dialect, PrintPunctuation, "punctuation"> {
+ let assemblyFormat = "`<` $value `>`";
+}
+
+#endif // MLIR_DIALECT_VECTOR_IR_VECTOR_ATTRIBUTES
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
index 4a624bd5f1ccdc8..fcf7eb4a616b073 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
@@ -14,6 +14,7 @@
#define MLIR_DIALECT_VECTOR_IR_VECTOROPS_H
#include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Vector/Interfaces/MaskableOpInterface.h"
#include "mlir/Dialect/Vector/Interfaces/MaskingOpInterface.h"
#include "mlir/IR/AffineMap.h"
@@ -31,10 +32,10 @@
#include "llvm/ADT/StringExtras.h"
// Pull in all enum type definitions and utility function declarations.
-#include "mlir/Dialect/Vector/IR/VectorOpsEnums.h.inc"
+#include "mlir/Dialect/Vector/IR/VectorEnums.h.inc"
#define GET_ATTRDEF_CLASSES
-#include "mlir/Dialect/Vector/IR/VectorOpsAttrDefs.h.inc"
+#include "mlir/Dialect/Vector/IR/VectorAttributes.h.inc"
namespace mlir {
class MLIRContext;
@@ -157,7 +158,7 @@ Value selectPassthru(OpBuilder &builder, Value mask, Value newValue,
} // namespace mlir
#define GET_OP_CLASSES
+#include "mlir/Dialect/Vector/IR/VectorDialect.h.inc"
#include "mlir/Dialect/Vector/IR/VectorOps.h.inc"
-#include "mlir/Dialect/Vector/IR/VectorOpsDialect.h.inc"
#endif // MLIR_DIALECT_VECTOR_IR_VECTOROPS_H
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 28b5864914f6920..ab77c31b418e537 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -10,9 +10,13 @@
//
//===----------------------------------------------------------------------===//
-#ifndef VECTOR_OPS
-#define VECTOR_OPS
+#ifndef MLIR_DIALECT_VECTOR_IR_VECTOR_OPS
+#define MLIR_DIALECT_VECTOR_IR_VECTOR_OPS
+include "Vector.td"
+include "VectorAttributes.td"
+include "mlir/Dialect/Arith/IR/ArithBase.td"
+include "mlir/Dialect/Arith/IR/ArithOpsInterfaces.td"
include "mlir/Dialect/Vector/Interfaces/MaskableOpInterface.td"
include "mlir/Dialect/Vector/Interfaces/MaskingOpInterface.td"
include "mlir/IR/EnumAttr.td"
@@ -23,69 +27,6 @@ include "mlir/Interfaces/SideEffectInterfaces.td"
include "mlir/Interfaces/VectorInterfaces.td"
include "mlir/Interfaces/ViewLikeInterface.td"
-def Vector_Dialect : Dialect {
- let name = "vector";
- let cppNamespace = "::mlir::vector";
-
- let useDefaultAttributePrinterParser = 1;
- let hasConstantMaterializer = 1;
- let dependentDialects = ["arith::ArithDialect"];
-}
-
-// Base class for Vector dialect ops.
-class Vector_Op<string mnemonic, list<Trait> traits = []> :
- Op<Vector_Dialect, mnemonic, traits>;
-
-// The "kind" of combining function for contractions and reductions.
-def COMBINING_KIND_ADD : I32BitEnumAttrCaseBit<"ADD", 0, "add">;
-def COMBINING_KIND_MUL : I32BitEnumAttrCaseBit<"MUL", 1, "mul">;
-def COMBINING_KIND_MINUI : I32BitEnumAttrCaseBit<"MINUI", 2, "minui">;
-def COMBINING_KIND_MINSI : I32BitEnumAttrCaseBit<"MINSI", 3, "minsi">;
-def COMBINING_KIND_MINF : I32BitEnumAttrCaseBit<"MINF", 4, "minf">;
-def COMBINING_KIND_MAXUI : I32BitEnumAttrCaseBit<"MAXUI", 5, "maxui">;
-def COMBINING_KIND_MAXSI : I32BitEnumAttrCaseBit<"MAXSI", 6, "maxsi">;
-def COMBINING_KIND_MAXF : I32BitEnumAttrCaseBit<"MAXF", 7, "maxf">;
-def COMBINING_KIND_AND : I32BitEnumAttrCaseBit<"AND", 8, "and">;
-def COMBINING_KIND_OR : I32BitEnumAttrCaseBit<"OR", 9, "or">;
-def COMBINING_KIND_XOR : I32BitEnumAttrCaseBit<"XOR", 10, "xor">;
-def COMBINING_KIND_MINIMUMF : I32BitEnumAttrCaseBit<"MINIMUMF", 11, "minimumf">;
-def COMBINING_KIND_MAXIMUMF : I32BitEnumAttrCaseBit<"MAXIMUMF", 12, "maximumf">;
-
-def CombiningKind : I32BitEnumAttr<
- "CombiningKind",
- "Kind of combining function for contractions and reductions",
- [COMBINING_KIND_ADD, COMBINING_KIND_MUL, COMBINING_KIND_MINUI,
- COMBINING_KIND_MINSI, COMBINING_KIND_MINF, COMBINING_KIND_MAXUI,
- COMBINING_KIND_MAXSI, COMBINING_KIND_MAXF, COMBINING_KIND_AND,
- COMBINING_KIND_OR, COMBINING_KIND_XOR,
- COMBINING_KIND_MAXIMUMF, COMBINING_KIND_MINIMUMF]> {
- let cppNamespace = "::mlir::vector";
- let genSpecializedAttr = 0;
-}
-
-/// An attribute that specifies the combining function for `vector.contract`,
-/// and `vector.reduction`.
-def Vector_CombiningKindAttr : EnumAttr<Vector_Dialect, CombiningKind, "kind"> {
- let assemblyFormat = "`<` $value `>`";
-}
-
-def Vector_IteratorType : I32EnumAttr<"IteratorType", "Iterator type", [
- I32EnumAttrCase<"parallel", 0>,
- I32EnumAttrCase<"reduction", 1>
-]> {
- let genSpecializedAttr = 0;
- let cppNamespace = "::mlir::vector";
-}
-
-def Vector_IteratorTypeEnum
- : EnumAttr<Vector_Dialect, Vector_IteratorType, "iterator_type"> {
- let assemblyFormat = "`<` $value `>`";
-}
-
-def Vector_IteratorTypeArrayAttr
- : TypedArrayAttrBase<Vector_IteratorTypeEnum,
- "Iterator type should be an enum.">;
-
// TODO: Add an attribute to specify a different algebra with operators other
// than the current set: {*, +}.
def Vector_ContractionOp :
@@ -274,12 +215,16 @@ def Vector_ReductionOp :
Vector_Op<"reduction", [Pure,
PredOpTrait<"source operand and result have same element type",
TCresVTEtIsSameAsOpBase<0, 0>>,
+ DeclareOpInterfaceMethods<ArithFastMathInterface>,
DeclareOpInterfaceMethods<MaskableOpInterface>,
- DeclareOpInterfaceMethods<VectorUnrollOpInterface,
- ["getShapeForUnroll"]>]>,
+ DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
+ ]>,
Arguments<(ins Vector_CombiningKindAttr:$kind,
AnyVectorOfAnyRank:$vector,
- Optional<AnyType>:$acc)>,
+ Optional<AnyType>:$acc,
+ DefaultValuedAttr<
+ Arith_FastMathAttr,
+ "::mlir::arith::FastMathFlags::none">:$fastmath)>,
Results<(outs AnyType:$dest)> {
let summary = "reduction operation";
let description = [{
@@ -309,9 +254,13 @@ def Vector_ReductionOp :
}];
let builders = [
// Builder that infers the type of `dest`.
- OpBuilder<(ins "CombiningKind":$kind, "Value":$vector, "Value":$acc)>,
+ OpBuilder<(ins "CombiningKind":$kind, "Value":$vector, "Value":$acc,
+ CArg<"::mlir::arith::FastMathFlags",
+ "::mlir::arith::FastMathFlags::none">:$fastMathFlags)>,
// Builder that infers the type of `dest` and has no accumulator.
- OpBuilder<(ins "CombiningKind":$kind, "Value":$vector)>
+ OpBuilder<(ins "CombiningKind":$kind, "Value":$vector,
+ CArg<"::mlir::arith::FastMathFlags",
+ "::mlir::arith::FastMathFlags::none">:$fastMathFlags)>
];
// TODO: Migrate to assemblyFormat once `AllTypesMatch` supports optional
@@ -2466,22 +2415,6 @@ def Vector_TransposeOp :
let hasVerifier = 1;
}
-def PrintPunctuation : I32EnumAttr<"PrintPunctuation",
- "Punctuation for separating vectors or vector elements", [
- I32EnumAttrCase<"NoPunctuation", 0, "no_punctuation">,
- I32EnumAttrCase<"NewLine", 1, "newline">,
- I32EnumAttrCase<"Comma", 2, "comma">,
- I32EnumAttrCase<"Open", 3, "open">,
- I32EnumAttrCase<"Close", 4, "close">
-]> {
- let cppNamespace = "::mlir::vector";
- let genSpecializedAttr = 0;
-}
-
-def Vector_PrintPunctuation : EnumAttr<Vector_Dialect, PrintPunctuation, "punctuation"> {
- let assemblyFormat = "`<` $value `>`";
-}
-
def Vector_PrintOp :
Vector_Op<"print", []>,
Arguments<(ins Optional<Type<Or<[
@@ -2936,4 +2869,4 @@ def Vector_WarpExecuteOnLane0Op : Vector_Op<"warp_execute_on_lane_0",
}];
}
-#endif // VECTOR_OPS
+#endif // MLIR_DIALECT_VECTOR_IR_VECTOR_OPS
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index a979237d1f63e17..2c08257fc3089b6 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -8,6 +8,7 @@
#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
+#include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
@@ -592,11 +593,11 @@ struct VectorToScalarMapper<LLVM::vector_reduce_fmin> {
} // namespace
template <class LLVMRedIntrinOp>
-static Value
-createFPReductionComparisonOpLowering(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Value vectorOperand, Value accumulator) {
- Value result = rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand);
+static Value createFPReductionComparisonOpLowering(
+ ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
+ Value vectorOperand, Value accumulator, LLVM::FastmathFlagsAttr fmf) {
+ Value result =
+ rewriter.create<LLVMRedIntrinOp>(loc, llvmType, vectorOperand, fmf);
if (accumulator) {
result =
@@ -641,87 +642,72 @@ static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
/// `fmaximum`/`fminimum`.
/// More information: https://github.com/llvm/llvm-project/issues/64940
template <class LLVMRedIntrinOp, class MaskNeutral>
-static Value lowerMaskedReductionWithRegular(
- ConversionPatternRewriter &rewriter, Location loc, Type llvmType,
- Value vectorOperand, Value accumulator, Value mask) {
+static Value
+lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType,
+ Value vectorOperand, Value accumulator,
+ Value mask, LLVM::FastmathFlagsAttr fmf) {
const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
rewriter, loc, llvmType, vectorOperand.getType());
const Value selectedVectorByMask = rewriter.create<LLVM::SelectOp>(
loc, mask, vectorOperand, vectorMaskNeutral);
return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
- rewriter, loc, llvmType, selectedVectorByMask, accumulator);
+ rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
}
-/// Overloaded methods to lower a reduction to an llvm instrinsic that requires
-/// a start value. This start value format spans across fp reductions without
-/// mask and all the masked reduction intrinsics.
-template <class LLVMVPRedIntrinOp, class ReductionNeutral>
-static Value lowerReductionWithStartValue(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Value vectorOperand,
- Value accumulator) {
- accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
- llvmType, accumulator);
- return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
- /*startValue=*/accumulator,
- vectorOperand);
-}
-
-template <class LLVMVPRedIntrinOp, class ReductionNeutral>
+template <class LLVMRedIntrinOp, class ReductionNeutral>
static Value
lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
Type llvmType, Value vectorOperand,
- Value accumulator, bool reassociateFPReds) {
+ Value accumulator, LLVM::FastmathFlagsAttr fmf) {
accumulator = getOrCreateAccumulator<ReductionNeutral>(rewriter, loc,
llvmType, accumulator);
- return rewriter.create<LLVMVPRedIntrinOp>(loc, llvmType,
- ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/66905
More information about the Mlir-commits
mailing list