[llvm] [SystemZ] Add realistic cost estimates for vector reduction intrinsics (PR #118319)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 3 05:49:43 PST 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-systemz
Author: Dominik Steenken (dominik-steenken)
<details>
<summary>Changes</summary>
This PR adds more realistic cost estimates for these reduction intrinsics
- `llvm.vector.reduce.umax`
- `llvm.vector.reduce.umin`
- `llvm.vector.reduce.smax`
- `llvm.vector.reduce.smin`
- `llvm.vector.reduce.fadd`
- `llvm.vector.reduce.fmul`
- `llvm.vector.reduce.fmax`
- `llvm.vector.reduce.fmin`
- `llvm.vector.reduce.fmaximum`
- `llvm.vector.reduce.fminimum`
- `llvm.vector.reduce.mul
`
The pre-existing cost estimates for `llvm.vector.reduce.add` are moved to `getArithmeticReductionCosts` to reduce complexity in `getVectorIntrinsicInstrCost` and enable other passes, like the SLP vectorizer, to benefit from these updated calculations.
These are not expected to provide noticable performance improvements and are rather provided for the sake of completeness and correctness. This PR is in draft mode pending benchmark confirmation of this.
This also provides and/or updates cost tests for all of these intrinsics.
This PR was co-authored by me and @<!-- -->JonPsson1 .
---
Patch is 88.65 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/118319.diff
7 Files Affected:
- (modified) llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp (+75-18)
- (modified) llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h (+7)
- (removed) llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll (-128)
- (added) llvm/test/Analysis/CostModel/SystemZ/vector-reductions.ll (+376)
- (added) llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll (+188)
- (added) llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll (+411)
- (added) llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll (+188)
``````````diff
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 83b42f6d1794d5..772efcdf8f9fc1 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -21,6 +21,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/MathExtras.h"
using namespace llvm;
@@ -1396,30 +1397,86 @@ InstructionCost SystemZTTIImpl::getInterleavedMemoryOpCost(
return NumVectorMemOps + NumPermutes;
}
+InstructionCost getIntAddReductionCost(unsigned NumVec, unsigned ScalarBits) {
+ InstructionCost Cost = 0;
+ // Binary Tree of N/2 + N/4 + ... operations yields N - 1 operations total.
+ Cost += NumVec - 1;
+ // For integer adds, VSUM creates shorter reductions on the final vector.
+ Cost += (ScalarBits < 32) ? 3 : 2;
+ return Cost;
+}
+
+InstructionCost getFastReductionCost(unsigned NumVec, unsigned NumElems,
+ unsigned ScalarBits) {
+ unsigned NumEltsPerVecReg = (SystemZ::VectorBits / ScalarBits);
+ InstructionCost Cost = 0;
+ // Binary Tree of N/2 + N/4 + ... operations yields N - 1 operations total.
+ Cost += NumVec - 1;
+ // For each shuffle / arithmetic layer, we need 2 instructions, and we need
+ // log2(Elements in Last Vector) layers.
+ Cost += 2 * Log2_32_Ceil(std::min(NumElems, NumEltsPerVecReg));
+ return Cost;
+}
+
+inline bool customCostReductions(unsigned Opcode) {
+ return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
+ Opcode == Instruction::Add || Opcode == Instruction::Mul;
+}
+
+InstructionCost
+SystemZTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+ std::optional<FastMathFlags> FMF,
+ TTI::TargetCostKind CostKind) {
+ unsigned ScalarBits = Ty->getScalarSizeInBits();
+ // The following is only for subtargets with vector math, non-ordered
+ // reductions, and reasonable scalar sizes for int and fp add/mul.
+ if (customCostReductions(Opcode) && ST->hasVector() &&
+ !TTI::requiresOrderedReduction(FMF) &&
+ ScalarBits <= SystemZ::VectorBits) {
+ unsigned NumVectors = getNumVectorRegs(Ty);
+ unsigned NumElems = ((FixedVectorType *)Ty)->getNumElements();
+ // Integer Add is using custom code gen, that needs to be accounted for.
+ if (Opcode == Instruction::Add)
+ return getIntAddReductionCost(NumVectors, ScalarBits);
+ // The base cost is the same across all other arithmetic instructions
+ InstructionCost Cost =
+ getFastReductionCost(NumVectors, NumElems, ScalarBits);
+ // But we need to account for the final op involving the scalar operand.
+ if ((Opcode == Instruction::FAdd) || (Opcode == Instruction::FMul))
+ Cost += 1;
+ return Cost;
+ }
+ // otherwise, fall back to the standard implementation
+ return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
+}
+
+InstructionCost
+SystemZTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
+ FastMathFlags FMF,
+ TTI::TargetCostKind CostKind) {
+ // Return custom costs only on subtargets with vector enhancements.
+ if (ST->hasVectorEnhancements1()) {
+ unsigned NumVectors = getNumVectorRegs(Ty);
+ unsigned NumElems = ((FixedVectorType *)Ty)->getNumElements();
+ unsigned ScalarBits = Ty->getScalarSizeInBits();
+ InstructionCost Cost = 0;
+ // Binary Tree of N/2 + N/4 + ... operations yields N - 1 operations total.
+ Cost += NumVectors - 1;
+ // For the final vector, we need shuffle + min/max operations, and
+ // we need #Elements - 1 of them.
+ Cost += 2 * (std::min(NumElems, SystemZ::VectorBits / ScalarBits) - 1);
+ return Cost;
+ }
+ // For other targets, fall back to the standard implementation
+ return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
+}
+
static int
getVectorIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
const SmallVectorImpl<Type *> &ParamTys) {
if (RetTy->isVectorTy() && ID == Intrinsic::bswap)
return getNumVectorRegs(RetTy); // VPERM
- if (ID == Intrinsic::vector_reduce_add) {
- // Retrieve number and size of elements for the vector op.
- auto *VTy = cast<FixedVectorType>(ParamTys.front());
- unsigned ScalarSize = VTy->getScalarSizeInBits();
- // For scalar sizes >128 bits, we fall back to the generic cost estimate.
- if (ScalarSize > SystemZ::VectorBits)
- return -1;
- // This many vector regs are needed to represent the input elements (V).
- unsigned VectorRegsNeeded = getNumVectorRegs(VTy);
- // This many instructions are needed for the final sum of vector elems (S).
- unsigned LastVectorHandling = (ScalarSize < 32) ? 3 : 2;
- // We use vector adds to create a sum vector, which takes
- // V/2 + V/4 + ... = V - 1 operations.
- // Then, we need S operations to sum up the elements of that sum vector,
- // for a total of V + S - 1 operations.
- int Cost = VectorRegsNeeded + LastVectorHandling - 1;
- return Cost;
- }
return -1;
}
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 6795da59bf5b16..512fcc854d532d 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -130,6 +130,13 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false);
+ InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+ std::optional<FastMathFlags> FMF,
+ TTI::TargetCostKind CostKind);
+ InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
+ FastMathFlags FMF,
+ TTI::TargetCostKind CostKind);
+
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
TTI::TargetCostKind CostKind);
diff --git a/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll b/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll
deleted file mode 100644
index 90b5b746c914ab..00000000000000
--- a/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll
+++ /dev/null
@@ -1,128 +0,0 @@
-; RUN: opt < %s -mtriple=systemz-unknown -mcpu=z13 -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s
-
-define void @reduce(ptr %src, ptr %dst) {
-; CHECK-LABEL: 'reduce'
-; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %R2_64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %V2_64)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R4_64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %V4_64)
-; CHECK: Cost Model: Found an estimated cost of 5 for instruction: %R8_64 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %V8_64)
-; CHECK: Cost Model: Found an estimated cost of 9 for instruction: %R16_64 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %V16_64)
-; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %R2_32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %V2_32)
-; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %R4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %V4_32)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %V8_32)
-; CHECK: Cost Model: Found an estimated cost of 5 for instruction: %R16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %V16_32)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %V2_16)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %V4_16)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %V8_16)
-; CHECK: Cost Model: Found an estimated cost of 4 for instruction: %R16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %V16_16)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %V2_8)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %V4_8)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %V8_8)
-; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %R16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %V16_8)
-;
-; CHECK: Cost Model: Found an estimated cost of 10 for instruction: %R128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> %V128_8)
-; CHECK: Cost Model: Found an estimated cost of 20 for instruction: %R4_256 = call i256 @llvm.vector.reduce.add.v4i256(<4 x i256> %V4_256)
-
- ; REDUCEADD64
-
- %V2_64 = load <2 x i64>, ptr %src, align 8
- %R2_64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %V2_64)
- store volatile i64 %R2_64, ptr %dst, align 4
-
- %V4_64 = load <4 x i64>, ptr %src, align 8
- %R4_64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %V4_64)
- store volatile i64 %R4_64, ptr %dst, align 4
-
- %V8_64 = load <8 x i64>, ptr %src, align 8
- %R8_64 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %V8_64)
- store volatile i64 %R8_64, ptr %dst, align 4
-
- %V16_64 = load <16 x i64>, ptr %src, align 8
- %R16_64 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %V16_64)
- store volatile i64 %R16_64, ptr %dst, align 4
-
- ; REDUCEADD32
-
- %V2_32 = load <2 x i32>, ptr %src, align 8
- %R2_32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %V2_32)
- store volatile i32 %R2_32, ptr %dst, align 4
-
- %V4_32 = load <4 x i32>, ptr %src, align 8
- %R4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %V4_32)
- store volatile i32 %R4_32, ptr %dst, align 4
-
- %V8_32 = load <8 x i32>, ptr %src, align 8
- %R8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %V8_32)
- store volatile i32 %R8_32, ptr %dst, align 4
-
- %V16_32 = load <16 x i32>, ptr %src, align 8
- %R16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %V16_32)
- store volatile i32 %R16_32, ptr %dst, align 4
-
- ; REDUCEADD16
-
- %V2_16 = load <2 x i16>, ptr %src, align 8
- %R2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %V2_16)
- store volatile i16 %R2_16, ptr %dst, align 4
-
- %V4_16 = load <4 x i16>, ptr %src, align 8
- %R4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %V4_16)
- store volatile i16 %R4_16, ptr %dst, align 4
-
- %V8_16 = load <8 x i16>, ptr %src, align 8
- %R8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %V8_16)
- store volatile i16 %R8_16, ptr %dst, align 4
-
- %V16_16 = load <16 x i16>, ptr %src, align 8
- %R16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %V16_16)
- store volatile i16 %R16_16, ptr %dst, align 4
-
- ; REDUCEADD8
-
- %V2_8 = load <2 x i8>, ptr %src, align 8
- %R2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %V2_8)
- store volatile i8 %R2_8, ptr %dst, align 4
-
- %V4_8 = load <4 x i8>, ptr %src, align 8
- %R4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %V4_8)
- store volatile i8 %R4_8, ptr %dst, align 4
-
- %V8_8 = load <8 x i8>, ptr %src, align 8
- %R8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %V8_8)
- store volatile i8 %R8_8, ptr %dst, align 4
-
- %V16_8 = load <16 x i8>, ptr %src, align 8
- %R16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %V16_8)
- store volatile i8 %R16_8, ptr %dst, align 4
-
- ; EXTREME VALUES
-
- %V128_8 = load <128 x i8>, ptr %src, align 8
- %R128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> %V128_8)
- store volatile i8 %R128_8, ptr %dst, align 4
-
- %V4_256 = load <4 x i256>, ptr %src, align 8
- %R4_256 = call i256 @llvm.vector.reduce.add.v4i256(<4 x i256> %V4_256)
- store volatile i256 %R4_256, ptr %dst, align 8
-
- ret void
-}
-
-declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)
-declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
-declare i16 @llvm.vector.reduce.add.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)
-declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.add.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
-
-declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
-declare i256 @llvm.vector.reduce.add.v4i256(<4 x i256>)
diff --git a/llvm/test/Analysis/CostModel/SystemZ/vector-reductions.ll b/llvm/test/Analysis/CostModel/SystemZ/vector-reductions.ll
new file mode 100644
index 00000000000000..0def20215e9889
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/SystemZ/vector-reductions.ll
@@ -0,0 +1,376 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes='print<cost-model>' -disable-output -mtriple=s390x-unknown-linux \
+; RUN: -mcpu=z15 < %s 2>&1 | FileCheck %s --check-prefix=Z15
+
+define void @fadd_reductions() {
+; Z15-LABEL: 'fadd_reductions'
+; Z15-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fadd_v8f32 = call float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fadd_v2f64 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f64 = call double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f128 = call fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.0, <4 x float> undef)
+ %fadd_v8f32 = call float @llvm.vector.reduce.fadd.v8f32(float 0.0, <8 x float> undef)
+ %fadd_v2f64 = call double @llvm.vector.reduce.fadd.v2f64(double 0.0, <2 x double> undef)
+ %fadd_v4f64 = call double @llvm.vector.reduce.fadd.v4f64(double 0.0, <4 x double> undef)
+ %fadd_v4f128 = call fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+ ret void
+}
+
+define void @fast_fadd_reductions(ptr %src, ptr %dst) {
+; Z15-LABEL: 'fast_fadd_reductions'
+; Z15-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %fadd_v4f32 = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %fadd_v8f32 = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %fadd_v2f64 = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fadd_v4f64 = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %fadd_v4f32 = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.0, <4 x float> undef)
+ %fadd_v8f32 = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.0, <8 x float> undef)
+ %fadd_v2f64 = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.0, <2 x double> undef)
+ %fadd_v4f64 = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.0, <4 x double> undef)
+ %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+ ret void
+}
+
+define void @fmul_reductions() {
+; Z15-LABEL: 'fmul_reductions'
+; Z15-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fmul_v4f32 = call float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fmul_v8f32 = call float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fmul_v2f64 = call double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fmul_v4f64 = call double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fmul_v4f128 = call fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %fmul_v4f32 = call float @llvm.vector.reduce.fmul.v4f32(float 0.0, <4 x float> undef)
+ %fmul_v8f32 = call float @llvm.vector.reduce.fmul.v8f32(float 0.0, <8 x float> undef)
+ %fmul_v2f64 = call double @llvm.vector.reduce.fmul.v2f64(double 0.0, <2 x double> undef)
+ %fmul_v4f64 = call double @llvm.vector.reduce.fmul.v4f64(double 0.0, <4 x double> undef)
+ %fmul_v4f128 = call fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+ ret void
+}
+
+define void @fast_fmul_reductions() {
+; Z15-LABEL: 'fast_fmul_reductions'
+; Z15-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %fmul_v4f32 = call fast float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %fmul_v8f32 = call fast float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %fmul_v2f64 = call fast double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fmul_v4f64 = call fast double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %fmul_v4f32 = call fast float @llvm.vector.reduce.fmul.v4f32(float 0.0, <4 x float> undef)
+ %fmul_v8f32 = call fast float @llvm.vector.reduce.fmul.v8f32(float 0.0, <8 x float> undef)
+ %fmul_v2f64 = call fast double @llvm.vector.reduce.fmul.v2f64(double 0.0, <2 x double> undef)
+ %fmul_v4f64 = call fast double @llvm.vector.reduce.fmul.v4f64(double 0.0, <4 x double> undef)
+ %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+
+ ret void
+}
+
+define void @fmin_reductions() {
+; Z15-LABEL: 'fmin_reductions'
+; Z15-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4f32 = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8f32 = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2f64 = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> undef)
+; Z15-NEXT: Cost Model: Found an estimated cost of 3 for instruction: ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/118319
More information about the llvm-commits
mailing list