[llvm] r362612 - [LoopUtils][SLPVectorizer] clean up management of fast-math-flags
Sanjay Patel via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 5 07:58:04 PDT 2019
Author: spatel
Date: Wed Jun 5 07:58:04 2019
New Revision: 362612
URL: http://llvm.org/viewvc/llvm-project?rev=362612&view=rev
Log:
[LoopUtils][SLPVectorizer] clean up management of fast-math-flags
Instead of passing around fast-math-flags as a parameter, we can set those
using an IRBuilder guard object. This is no-functional-change-intended.
The motivation is to eventually fix the vectorizers to use and set the
correct fast-math-flags for reductions. Examples of that not behaving as
expected are:
https://bugs.llvm.org/show_bug.cgi?id=23116 (should be able to reduce with less than 'fast')
https://bugs.llvm.org/show_bug.cgi?id=35538 (possible miscompile for -0.0)
D61802 (should be able to reduce with IR-level FMF)
Differential Revision: https://reviews.llvm.org/D62272
Modified:
llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h
llvm/trunk/lib/CodeGen/ExpandReductions.cpp
llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp
llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp
Modified: llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h?rev=362612&r1=362611&r2=362612&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h (original)
+++ llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h Wed Jun 5 07:58:04 2019
@@ -299,10 +299,10 @@ getOrderedReduction(IRBuilder<> &Builder
ArrayRef<Value *> RedOps = None);
/// Generates a vector reduction using shufflevectors to reduce the value.
+/// Fast-math-flags are propagated using the IRBuilder's setting.
Value *getShuffleReduction(IRBuilder<> &Builder, Value *Src, unsigned Op,
RecurrenceDescriptor::MinMaxRecurrenceKind
MinMaxKind = RecurrenceDescriptor::MRK_Invalid,
- FastMathFlags FMF = FastMathFlags(),
ArrayRef<Value *> RedOps = None);
/// Create a target reduction of the given vector. The reduction operation
@@ -310,17 +310,18 @@ Value *getShuffleReduction(IRBuilder<> &
/// additional information supplied in \p Flags.
/// The target is queried to determine if intrinsics or shuffle sequences are
/// required to implement the reduction.
+/// Fast-math-flags are propagated using the IRBuilder's setting.
Value *createSimpleTargetReduction(IRBuilder<> &B,
const TargetTransformInfo *TTI,
unsigned Opcode, Value *Src,
TargetTransformInfo::ReductionFlags Flags =
TargetTransformInfo::ReductionFlags(),
- FastMathFlags FMF = FastMathFlags(),
ArrayRef<Value *> RedOps = None);
/// Create a generic target reduction using a recurrence descriptor \p Desc
/// The target is queried to determine if intrinsics or shuffle sequences are
/// required to implement the reduction.
+/// Fast-math-flags are propagated using the RecurrenceDescriptor.
Value *createTargetReduction(IRBuilder<> &B, const TargetTransformInfo *TTI,
RecurrenceDescriptor &Desc, Value *Src,
bool NoNaN = false);
Modified: llvm/trunk/lib/CodeGen/ExpandReductions.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/ExpandReductions.cpp?rev=362612&r1=362611&r2=362612&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/ExpandReductions.cpp (original)
+++ llvm/trunk/lib/CodeGen/ExpandReductions.cpp Wed Jun 5 07:58:04 2019
@@ -118,11 +118,14 @@ bool expandReductions(Function &F, const
}
if (!TTI->shouldExpandReduction(II))
continue;
+ // Propagate FMF using the builder.
FastMathFlags FMF =
isa<FPMathOperator>(II) ? II->getFastMathFlags() : FastMathFlags{};
+ IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
+ Builder.setFastMathFlags(FMF);
Value *Rdx =
IsOrdered ? getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK)
- : getShuffleReduction(Builder, Vec, getOpcode(ID), MRK, FMF);
+ : getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
II->replaceAllUsesWith(Rdx);
II->eraseFromParent();
Changed = true;
Modified: llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp?rev=362612&r1=362611&r2=362612&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp (original)
+++ llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp Wed Jun 5 07:58:04 2019
@@ -675,12 +675,6 @@ bool llvm::hasIterationCountInvariantInP
return true;
}
-static Value *addFastMathFlag(Value *V, FastMathFlags FMF) {
- if (isa<FPMathOperator>(V))
- cast<Instruction>(V)->setFastMathFlags(FMF);
- return V;
-}
-
Value *llvm::createMinMaxOp(IRBuilder<> &Builder,
RecurrenceDescriptor::MinMaxRecurrenceKind RK,
Value *Left, Value *Right) {
@@ -761,7 +755,7 @@ llvm::getOrderedReduction(IRBuilder<> &B
Value *
llvm::getShuffleReduction(IRBuilder<> &Builder, Value *Src, unsigned Op,
RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
- FastMathFlags FMF, ArrayRef<Value *> RedOps) {
+ ArrayRef<Value *> RedOps) {
unsigned VF = Src->getType()->getVectorNumElements();
// VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
// and vector ops, reducing the set of values being computed by half each
@@ -784,10 +778,9 @@ llvm::getShuffleReduction(IRBuilder<> &B
ConstantVector::get(ShuffleMask), "rdx.shuf");
if (Op != Instruction::ICmp && Op != Instruction::FCmp) {
- // Floating point operations had to be 'fast' to enable the reduction.
- TmpVec = addFastMathFlag(Builder.CreateBinOp((Instruction::BinaryOps)Op,
- TmpVec, Shuf, "bin.rdx"),
- FMF);
+ // The builder propagates its fast-math-flags setting.
+ TmpVec = Builder.CreateBinOp((Instruction::BinaryOps)Op, TmpVec, Shuf,
+ "bin.rdx");
} else {
assert(MinMaxKind != RecurrenceDescriptor::MRK_Invalid &&
"Invalid min/max");
@@ -804,7 +797,7 @@ llvm::getShuffleReduction(IRBuilder<> &B
/// flags (if generating min/max reductions).
Value *llvm::createSimpleTargetReduction(
IRBuilder<> &Builder, const TargetTransformInfo *TTI, unsigned Opcode,
- Value *Src, TargetTransformInfo::ReductionFlags Flags, FastMathFlags FMF,
+ Value *Src, TargetTransformInfo::ReductionFlags Flags,
ArrayRef<Value *> RedOps) {
assert(isa<VectorType>(Src->getType()) && "Type must be a vector");
@@ -874,7 +867,7 @@ Value *llvm::createSimpleTargetReduction
}
if (TTI->useReductionIntrinsic(Opcode, Src->getType(), Flags))
return BuildFunc();
- return getShuffleReduction(Builder, Src, Opcode, MinMaxKind, FMF, RedOps);
+ return getShuffleReduction(Builder, Src, Opcode, MinMaxKind, RedOps);
}
/// Create a vector reduction using a given recurrence descriptor.
@@ -887,39 +880,36 @@ Value *llvm::createTargetReduction(IRBui
RD::RecurrenceKind RecKind = Desc.getRecurrenceKind();
TargetTransformInfo::ReductionFlags Flags;
Flags.NoNaN = NoNaN;
+
+ // All ops in the reduction inherit fast-math-flags from the recurrence
+ // descriptor.
+ IRBuilder<>::FastMathFlagGuard FMFGuard(B);
+ B.setFastMathFlags(Desc.getFastMathFlags());
+
switch (RecKind) {
case RD::RK_FloatAdd:
- return createSimpleTargetReduction(B, TTI, Instruction::FAdd, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::FAdd, Src, Flags);
case RD::RK_FloatMult:
- return createSimpleTargetReduction(B, TTI, Instruction::FMul, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::FMul, Src, Flags);
case RD::RK_IntegerAdd:
- return createSimpleTargetReduction(B, TTI, Instruction::Add, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::Add, Src, Flags);
case RD::RK_IntegerMult:
- return createSimpleTargetReduction(B, TTI, Instruction::Mul, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::Mul, Src, Flags);
case RD::RK_IntegerAnd:
- return createSimpleTargetReduction(B, TTI, Instruction::And, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::And, Src, Flags);
case RD::RK_IntegerOr:
- return createSimpleTargetReduction(B, TTI, Instruction::Or, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::Or, Src, Flags);
case RD::RK_IntegerXor:
- return createSimpleTargetReduction(B, TTI, Instruction::Xor, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::Xor, Src, Flags);
case RD::RK_IntegerMinMax: {
RD::MinMaxRecurrenceKind MMKind = Desc.getMinMaxRecurrenceKind();
Flags.IsMaxOp = (MMKind == RD::MRK_SIntMax || MMKind == RD::MRK_UIntMax);
Flags.IsSigned = (MMKind == RD::MRK_SIntMax || MMKind == RD::MRK_SIntMin);
- return createSimpleTargetReduction(B, TTI, Instruction::ICmp, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::ICmp, Src, Flags);
}
case RD::RK_FloatMinMax: {
Flags.IsMaxOp = Desc.getMinMaxRecurrenceKind() == RD::MRK_FloatMax;
- return createSimpleTargetReduction(B, TTI, Instruction::FCmp, Src, Flags,
- Desc.getFastMathFlags());
+ return createSimpleTargetReduction(B, TTI, Instruction::FCmp, Src, Flags);
}
default:
llvm_unreachable("Unhandled RecKind");
Modified: llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp?rev=362612&r1=362611&r2=362612&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp Wed Jun 5 07:58:04 2019
@@ -6105,6 +6105,9 @@ public:
unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
Value *VectorizedTree = nullptr;
+
+ // FIXME: Fast-math-flags should be set based on the instructions in the
+ // reduction (not all of 'fast' are required).
IRBuilder<> Builder(cast<Instruction>(ReductionRoot));
FastMathFlags Unsafe;
Unsafe.setFast();
@@ -6294,11 +6297,14 @@ private:
assert(isPowerOf2_32(ReduxWidth) &&
"We only handle power-of-two reductions for now");
- if (!IsPairwiseReduction)
+ if (!IsPairwiseReduction) {
+ // FIXME: The builder should use an FMF guard. It should not be hard-coded
+ // to 'fast'.
+ assert(Builder.getFastMathFlags().isFast() && "Expected 'fast' FMF");
return createSimpleTargetReduction(
Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
- ReductionData.getFlags(), FastMathFlags::getFast(),
- ReductionOps.back());
+ ReductionData.getFlags(), ReductionOps.back());
+ }
Value *TmpVec = VectorizedValue;
for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
More information about the llvm-commits
mailing list