[llvm-branch-commits] [llvm] c74e853 - [Analysis] flatten enums for recurrence types
Sanjay Patel via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jan 1 09:50:32 PST 2021
Author: Sanjay Patel
Date: 2021-01-01T12:20:16-05:00
New Revision: c74e8539ff372a89d08e7bfea7323a4dc2979d22
URL: https://github.com/llvm/llvm-project/commit/c74e8539ff372a89d08e7bfea7323a4dc2979d22
DIFF: https://github.com/llvm/llvm-project/commit/c74e8539ff372a89d08e7bfea7323a4dc2979d22.diff
LOG: [Analysis] flatten enums for recurrence types
This is almost all mechanical search-and-replace and
no-functional-change-intended (NFC). Having a single
enum makes it easier to match/reason about the
reduction cases.
The goal is to remove `Opcode` from reduction matching
code in the vectorizers because that makes it harder to
adapt the code to handle intrinsics.
The code in RecurrenceDescriptor::AddReductionVar() is
the only place that required closer inspection. It uses
a RecurrenceDescriptor and a second InstDesc to sometimes
overwrite part of the struct. It seem like we should be
able to simplify that logic, but it's not clear exactly
which cmp+sel patterns that we are trying to handle/avoid.
Added:
Modified:
llvm/include/llvm/Analysis/IVDescriptors.h
llvm/include/llvm/Transforms/Utils/LoopUtils.h
llvm/lib/Analysis/IVDescriptors.cpp
llvm/lib/CodeGen/ExpandReductions.cpp
llvm/lib/Transforms/Utils/LoopUtils.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index b9f6b7c2d04e..798eb430df08 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -34,6 +34,24 @@ class SCEV;
class DominatorTree;
class ICFLoopSafetyInfo;
+/// These are the kinds of recurrences that we support.
+enum class RecurKind {
+ None, ///< Not a recurrence.
+ Add, ///< Sum of integers.
+ Mul, ///< Product of integers.
+ Or, ///< Bitwise or logical OR of integers.
+ And, ///< Bitwise or logical AND of integers.
+ Xor, ///< Bitwise or logical XOR of integers.
+ SMin, ///< Signed integer min implemented in terms of select(cmp()).
+ SMax, ///< Signed integer max implemented in terms of select(cmp()).
+ UMin, ///< Unisgned integer min implemented in terms of select(cmp()).
+ UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
+ FAdd, ///< Sum of floats.
+ FMul, ///< Product of floats.
+ FMin, ///< FP min implemented in terms of select(cmp()).
+ FMax ///< FP max implemented in terms of select(cmp()).
+};
+
/// The RecurrenceDescriptor is used to identify recurrences variables in a
/// loop. Reduction is a special case of recurrence that has uses of the
/// recurrence variable outside the loop. The method isReductionPHI identifies
@@ -48,40 +66,13 @@ class ICFLoopSafetyInfo;
/// This struct holds information about recurrence variables.
class RecurrenceDescriptor {
public:
- /// This enum represents the kinds of recurrences that we support.
- enum RecurrenceKind {
- RK_NoRecurrence, ///< Not a recurrence.
- RK_IntegerAdd, ///< Sum of integers.
- RK_IntegerMult, ///< Product of integers.
- RK_IntegerOr, ///< Bitwise or logical OR of numbers.
- RK_IntegerAnd, ///< Bitwise or logical AND of numbers.
- RK_IntegerXor, ///< Bitwise or logical XOR of numbers.
- RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()).
- RK_FloatAdd, ///< Sum of floats.
- RK_FloatMult, ///< Product of floats.
- RK_FloatMinMax ///< Min/max implemented in terms of select(cmp()).
- };
-
- // This enum represents the kind of minmax recurrence.
- enum MinMaxRecurrenceKind {
- MRK_Invalid,
- MRK_UIntMin,
- MRK_UIntMax,
- MRK_SIntMin,
- MRK_SIntMax,
- MRK_FloatMin,
- MRK_FloatMax
- };
-
RecurrenceDescriptor() = default;
- RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurrenceKind K,
- FastMathFlags FMF, MinMaxRecurrenceKind MK,
- Instruction *UAI, Type *RT, bool Signed,
- SmallPtrSetImpl<Instruction *> &CI)
+ RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurKind K,
+ FastMathFlags FMF, Instruction *UAI, Type *RT,
+ bool Signed, SmallPtrSetImpl<Instruction *> &CI)
: StartValue(Start), LoopExitInstr(Exit), Kind(K), FMF(FMF),
- MinMaxKind(MK), UnsafeAlgebraInst(UAI), RecurrenceType(RT),
- IsSigned(Signed) {
+ UnsafeAlgebraInst(UAI), RecurrenceType(RT), IsSigned(Signed) {
CastInsts.insert(CI.begin(), CI.end());
}
@@ -89,11 +80,11 @@ class RecurrenceDescriptor {
class InstDesc {
public:
InstDesc(bool IsRecur, Instruction *I, Instruction *UAI = nullptr)
- : IsRecurrence(IsRecur), PatternLastInst(I), MinMaxKind(MRK_Invalid),
- UnsafeAlgebraInst(UAI) {}
+ : IsRecurrence(IsRecur), PatternLastInst(I),
+ RecKind(RecurKind::None), UnsafeAlgebraInst(UAI) {}
- InstDesc(Instruction *I, MinMaxRecurrenceKind K, Instruction *UAI = nullptr)
- : IsRecurrence(true), PatternLastInst(I), MinMaxKind(K),
+ InstDesc(Instruction *I, RecurKind K, Instruction *UAI = nullptr)
+ : IsRecurrence(true), PatternLastInst(I), RecKind(K),
UnsafeAlgebraInst(UAI) {}
bool isRecurrence() const { return IsRecurrence; }
@@ -102,7 +93,7 @@ class RecurrenceDescriptor {
Instruction *getUnsafeAlgebraInst() const { return UnsafeAlgebraInst; }
- MinMaxRecurrenceKind getMinMaxKind() const { return MinMaxKind; }
+ RecurKind getRecKind() const { return RecKind; }
Instruction *getPatternInst() const { return PatternLastInst; }
@@ -112,8 +103,8 @@ class RecurrenceDescriptor {
// The last instruction in a min/max pattern (select of the select(icmp())
// pattern), or the current recurrence instruction otherwise.
Instruction *PatternLastInst;
- // If this is a min/max pattern the comparison predicate.
- MinMaxRecurrenceKind MinMaxKind;
+ // If this is a min/max pattern.
+ RecurKind RecKind;
// Recurrence has unsafe algebra.
Instruction *UnsafeAlgebraInst;
};
@@ -123,7 +114,7 @@ class RecurrenceDescriptor {
/// select(icmp()) this function advances the instruction pointer 'I' from the
/// compare instruction to the select instruction and stores this pointer in
/// 'PatternLastInst' member of the returned struct.
- static InstDesc isRecurrenceInstr(Instruction *I, RecurrenceKind Kind,
+ static InstDesc isRecurrenceInstr(Instruction *I, RecurKind Kind,
InstDesc &Prev, bool HasFunNoNaNAttr);
/// Returns true if instruction I has multiple uses in Insts
@@ -143,21 +134,20 @@ class RecurrenceDescriptor {
/// Returns a struct describing if the instruction is a
/// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
- static InstDesc isConditionalRdxPattern(RecurrenceKind Kind, Instruction *I);
+ static InstDesc isConditionalRdxPattern(RecurKind Kind, Instruction *I);
/// Returns identity corresponding to the RecurrenceKind.
- static Constant *getRecurrenceIdentity(RecurrenceKind K,
- MinMaxRecurrenceKind MK, Type *Tp);
+ static Constant *getRecurrenceIdentity(RecurKind K, Type *Tp);
/// Returns the opcode of binary operation corresponding to the
/// RecurrenceKind.
- static unsigned getRecurrenceBinOp(RecurrenceKind Kind);
+ static unsigned getRecurrenceBinOp(RecurKind Kind);
/// Returns true if Phi is a reduction of type Kind and adds it to the
/// RecurrenceDescriptor. If either \p DB is non-null or \p AC and \p DT are
/// non-null, the minimal bit width needed to compute the reduction will be
/// computed.
- static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop,
+ static bool AddReductionVar(PHINode *Phi, RecurKind Kind, Loop *TheLoop,
bool HasFunNoNaNAttr,
RecurrenceDescriptor &RedDes,
DemandedBits *DB = nullptr,
@@ -186,14 +176,12 @@ class RecurrenceDescriptor {
DenseMap<Instruction *, Instruction *> &SinkAfter,
DominatorTree *DT);
- RecurrenceKind getRecurrenceKind() const { return Kind; }
+ RecurKind getRecurrenceKind() const { return Kind; }
unsigned getRecurrenceBinOp() const {
return getRecurrenceBinOp(getRecurrenceKind());
}
- MinMaxRecurrenceKind getMinMaxRecurrenceKind() const { return MinMaxKind; }
-
FastMathFlags getFastMathFlags() const { return FMF; }
TrackingVH<Value> getRecurrenceStartValue() const { return StartValue; }
@@ -208,13 +196,29 @@ class RecurrenceDescriptor {
Instruction *getUnsafeAlgebraInst() const { return UnsafeAlgebraInst; }
/// Returns true if the recurrence kind is an integer kind.
- static bool isIntegerRecurrenceKind(RecurrenceKind Kind);
+ static bool isIntegerRecurrenceKind(RecurKind Kind);
/// Returns true if the recurrence kind is a floating point kind.
- static bool isFloatingPointRecurrenceKind(RecurrenceKind Kind);
+ static bool isFloatingPointRecurrenceKind(RecurKind Kind);
/// Returns true if the recurrence kind is an arithmetic kind.
- static bool isArithmeticRecurrenceKind(RecurrenceKind Kind);
+ static bool isArithmeticRecurrenceKind(RecurKind Kind);
+
+ /// Returns true if the recurrence kind is an integer min/max kind.
+ static bool isIntMinMaxRecurrenceKind(RecurKind Kind) {
+ return Kind == RecurKind::UMin || Kind == RecurKind::UMax ||
+ Kind == RecurKind::SMin || Kind == RecurKind::SMax;
+ }
+
+ /// Returns true if the recurrence kind is a floating-point min/max kind.
+ static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
+ return Kind == RecurKind::FMin || Kind == RecurKind::FMax;
+ }
+
+ /// Returns true if the recurrence kind is any min/max kind.
+ static bool isMinMaxRecurrenceKind(RecurKind Kind) {
+ return isIntMinMaxRecurrenceKind(Kind) || isFPMinMaxRecurrenceKind(Kind);
+ }
/// Returns the type of the recurrence. This type can be narrower than the
/// actual type of the Phi if the recurrence has been type-promoted.
@@ -239,12 +243,10 @@ class RecurrenceDescriptor {
// The instruction who's value is used outside the loop.
Instruction *LoopExitInstr = nullptr;
// The kind of the recurrence.
- RecurrenceKind Kind = RK_NoRecurrence;
+ RecurKind Kind = RecurKind::None;
// The fast-math flags on the recurrent instructions. We propagate these
// fast-math flags into the vectorized FP instructions we generate.
FastMathFlags FMF;
- // If this a min/max recurrence the kind of recurrence.
- MinMaxRecurrenceKind MinMaxKind = MRK_Invalid;
// First occurrence of unasfe algebra in the PHI's use-chain.
Instruction *UnsafeAlgebraInst = nullptr;
// The type of the recurrence.
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index ba2bb0a4c6b0..b29add4cba0e 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -345,34 +345,31 @@ bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
OptimizationRemarkEmitter *ORE = nullptr);
/// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
-Value *createMinMaxOp(IRBuilderBase &Builder,
- RecurrenceDescriptor::MinMaxRecurrenceKind RK,
- Value *Left, Value *Right);
+Value *createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
+ Value *Right);
/// Generates an ordered vector reduction using extracts to reduce the value.
-Value *
-getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src, unsigned Op,
- RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =
- RecurrenceDescriptor::MRK_Invalid,
- ArrayRef<Value *> RedOps = None);
+Value *getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
+ unsigned Op, RecurKind MinMaxKind = RecurKind::None,
+ ArrayRef<Value *> RedOps = None);
/// Generates a vector reduction using shufflevectors to reduce the value.
/// Fast-math-flags are propagated using the IRBuilder's setting.
Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
- RecurrenceDescriptor::MinMaxRecurrenceKind
- MinMaxKind = RecurrenceDescriptor::MRK_Invalid,
+ RecurKind MinMaxKind = RecurKind::None,
ArrayRef<Value *> RedOps = None);
/// Create a target reduction of the given vector. The reduction operation
/// is described by the \p Opcode parameter. min/max reductions require
-/// additional information supplied in \p MinMaxKind.
+/// additional information supplied in \p RdxKind.
/// The target is queried to determine if intrinsics or shuffle sequences are
/// required to implement the reduction.
/// Fast-math-flags are propagated using the IRBuilder's setting.
-Value *createSimpleTargetReduction(
- IRBuilderBase &B, const TargetTransformInfo *TTI, unsigned Opcode,
- Value *Src, RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
- ArrayRef<Value *> RedOps = None);
+Value *createSimpleTargetReduction(IRBuilderBase &B,
+ const TargetTransformInfo *TTI,
+ unsigned Opcode, Value *Src,
+ RecurKind RdxKind,
+ ArrayRef<Value *> RedOps = None);
/// Create a generic target reduction using a recurrence descriptor \p Desc
/// The target is queried to determine if intrinsics or shuffle sequences are
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index eac6f3cb30f8..0bd4f9854158 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -47,33 +47,36 @@ bool RecurrenceDescriptor::areAllUsesIn(Instruction *I,
return true;
}
-bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurrenceKind Kind) {
+bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
switch (Kind) {
default:
break;
- case RK_IntegerAdd:
- case RK_IntegerMult:
- case RK_IntegerOr:
- case RK_IntegerAnd:
- case RK_IntegerXor:
- case RK_IntegerMinMax:
+ case RecurKind::Add:
+ case RecurKind::Mul:
+ case RecurKind::Or:
+ case RecurKind::And:
+ case RecurKind::Xor:
+ case RecurKind::SMax:
+ case RecurKind::SMin:
+ case RecurKind::UMax:
+ case RecurKind::UMin:
return true;
}
return false;
}
-bool RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurrenceKind Kind) {
- return (Kind != RK_NoRecurrence) && !isIntegerRecurrenceKind(Kind);
+bool RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurKind Kind) {
+ return (Kind != RecurKind::None) && !isIntegerRecurrenceKind(Kind);
}
-bool RecurrenceDescriptor::isArithmeticRecurrenceKind(RecurrenceKind Kind) {
+bool RecurrenceDescriptor::isArithmeticRecurrenceKind(RecurKind Kind) {
switch (Kind) {
default:
break;
- case RK_IntegerAdd:
- case RK_IntegerMult:
- case RK_FloatAdd:
- case RK_FloatMult:
+ case RecurKind::Add:
+ case RecurKind::Mul:
+ case RecurKind::FAdd:
+ case RecurKind::FMul:
return true;
}
return false;
@@ -186,7 +189,7 @@ static void collectCastsToIgnore(Loop *TheLoop, Instruction *Exit,
}
}
-bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind,
+bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind,
Loop *TheLoop, bool HasFunNoNaNAttr,
RecurrenceDescriptor &RedDes,
DemandedBits *DB,
@@ -302,29 +305,35 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind,
// FIXME: FMF is allowed on phi, but propagation is not handled correctly.
if (isa<FPMathOperator>(ReduxDesc.getPatternInst()) && !IsAPhi)
FMF &= ReduxDesc.getPatternInst()->getFastMathFlags();
+ // Update this reduction kind if we matched a new instruction.
+ // TODO: Can we eliminate the need for a 2nd InstDesc by keeping 'Kind'
+ // state accurate while processing the worklist?
+ if (ReduxDesc.getRecKind() != RecurKind::None)
+ Kind = ReduxDesc.getRecKind();
}
bool IsASelect = isa<SelectInst>(Cur);
// A conditional reduction operation must only have 2 or less uses in
// VisitedInsts.
- if (IsASelect && (Kind == RK_FloatAdd || Kind == RK_FloatMult) &&
+ if (IsASelect && (Kind == RecurKind::FAdd || Kind == RecurKind::FMul) &&
hasMultipleUsesOf(Cur, VisitedInsts, 2))
return false;
// A reduction operation must only have one use of the reduction value.
- if (!IsAPhi && !IsASelect && Kind != RK_IntegerMinMax &&
- Kind != RK_FloatMinMax && hasMultipleUsesOf(Cur, VisitedInsts, 1))
+ if (!IsAPhi && !IsASelect && !isMinMaxRecurrenceKind(Kind) &&
+ hasMultipleUsesOf(Cur, VisitedInsts, 1))
return false;
// All inputs to a PHI node must be a reduction value.
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
return false;
- if (Kind == RK_IntegerMinMax &&
+ if (isIntMinMaxRecurrenceKind(Kind) &&
(isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
++NumCmpSelectPatternInst;
- if (Kind == RK_FloatMinMax && (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
+ if (isFPMinMaxRecurrenceKind(Kind) &&
+ (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
++NumCmpSelectPatternInst;
// Check whether we found a reduction operator.
@@ -389,8 +398,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind,
// This means we have seen one but not the other instruction of the
// pattern or more than just a select and cmp.
- if ((Kind == RK_IntegerMinMax || Kind == RK_FloatMinMax) &&
- NumCmpSelectPatternInst != 2)
+ if (isMinMaxRecurrenceKind(Kind) && NumCmpSelectPatternInst != 2)
return false;
if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction)
@@ -408,7 +416,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind,
// can be ignore in the cost model. If we compute a
diff erent type than we
// did when evaluating the 'and', the 'and' will not be eliminated, and we
// will end up with
diff erent kinds of operations in the recurrence
- // expression (e.g., RK_IntegerAND, RK_IntegerADD). We give up if this is
+ // expression (e.g., IntegerAND, IntegerADD). We give up if this is
// the case.
//
// The vectorizer relies on InstCombine to perform the actual
@@ -448,9 +456,9 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind,
// is saved as part of the RecurrenceDescriptor.
// Save the description of this reduction variable.
- RecurrenceDescriptor RD(
- RdxStart, ExitInstruction, Kind, FMF, ReduxDesc.getMinMaxKind(),
- ReduxDesc.getUnsafeAlgebraInst(), RecurrenceType, IsSigned, CastInsts);
+ RecurrenceDescriptor RD(RdxStart, ExitInstruction, Kind, FMF,
+ ReduxDesc.getUnsafeAlgebraInst(), RecurrenceType,
+ IsSigned, CastInsts);
RedDes = RD;
return true;
@@ -467,7 +475,7 @@ RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I,
CmpInst::Predicate Pred;
if (match(I, m_OneUse(m_Cmp(Pred, m_Value(), m_Value())))) {
if (auto *Select = dyn_cast<SelectInst>(*I->user_begin()))
- return InstDesc(Select, Prev.getMinMaxKind());
+ return InstDesc(Select, Prev.getRecKind());
}
// Only match select with single use cmp condition.
@@ -477,21 +485,21 @@ RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I,
// Look for a min/max pattern.
if (match(I, m_UMin(m_Value(), m_Value())))
- return InstDesc(I, MRK_UIntMin);
+ return InstDesc(I, RecurKind::UMin);
if (match(I, m_UMax(m_Value(), m_Value())))
- return InstDesc(I, MRK_UIntMax);
+ return InstDesc(I, RecurKind::UMax);
if (match(I, m_SMax(m_Value(), m_Value())))
- return InstDesc(I, MRK_SIntMax);
+ return InstDesc(I, RecurKind::SMax);
if (match(I, m_SMin(m_Value(), m_Value())))
- return InstDesc(I, MRK_SIntMin);
+ return InstDesc(I, RecurKind::SMin);
if (match(I, m_OrdFMin(m_Value(), m_Value())))
- return InstDesc(I, MRK_FloatMin);
+ return InstDesc(I, RecurKind::FMin);
if (match(I, m_OrdFMax(m_Value(), m_Value())))
- return InstDesc(I, MRK_FloatMax);
+ return InstDesc(I, RecurKind::FMax);
if (match(I, m_UnordFMin(m_Value(), m_Value())))
- return InstDesc(I, MRK_FloatMin);
+ return InstDesc(I, RecurKind::FMin);
if (match(I, m_UnordFMax(m_Value(), m_Value())))
- return InstDesc(I, MRK_FloatMax);
+ return InstDesc(I, RecurKind::FMax);
return InstDesc(false, I);
}
@@ -506,8 +514,7 @@ RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I,
/// %add = fadd %0, %sum.1
/// %sum.2 = select %cmp, %add, %sum.1
RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isConditionalRdxPattern(
- RecurrenceKind Kind, Instruction *I) {
+RecurrenceDescriptor::isConditionalRdxPattern(RecurKind Kind, Instruction *I) {
SelectInst *SI = dyn_cast<SelectInst>(I);
if (!SI)
return InstDesc(false, I);
@@ -535,16 +542,16 @@ RecurrenceDescriptor::isConditionalRdxPattern(
if ((m_FAdd(m_Value(Op1), m_Value(Op2)).match(I1) ||
m_FSub(m_Value(Op1), m_Value(Op2)).match(I1)) &&
I1->isFast())
- return InstDesc(Kind == RK_FloatAdd, SI);
+ return InstDesc(Kind == RecurKind::FAdd, SI);
if (m_FMul(m_Value(Op1), m_Value(Op2)).match(I1) && (I1->isFast()))
- return InstDesc(Kind == RK_FloatMult, SI);
+ return InstDesc(Kind == RecurKind::FMul, SI);
return InstDesc(false, I);
}
RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind,
+RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurKind Kind,
InstDesc &Prev, bool HasFunNoNaNAttr) {
Instruction *UAI = Prev.getUnsafeAlgebraInst();
if (!UAI && isa<FPMathOperator>(I) && !I->hasAllowReassoc())
@@ -554,32 +561,32 @@ RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind,
default:
return InstDesc(false, I);
case Instruction::PHI:
- return InstDesc(I, Prev.getMinMaxKind(), Prev.getUnsafeAlgebraInst());
+ return InstDesc(I, Prev.getRecKind(), Prev.getUnsafeAlgebraInst());
case Instruction::Sub:
case Instruction::Add:
- return InstDesc(Kind == RK_IntegerAdd, I);
+ return InstDesc(Kind == RecurKind::Add, I);
case Instruction::Mul:
- return InstDesc(Kind == RK_IntegerMult, I);
+ return InstDesc(Kind == RecurKind::Mul, I);
case Instruction::And:
- return InstDesc(Kind == RK_IntegerAnd, I);
+ return InstDesc(Kind == RecurKind::And, I);
case Instruction::Or:
- return InstDesc(Kind == RK_IntegerOr, I);
+ return InstDesc(Kind == RecurKind::Or, I);
case Instruction::Xor:
- return InstDesc(Kind == RK_IntegerXor, I);
+ return InstDesc(Kind == RecurKind::Xor, I);
case Instruction::FDiv:
case Instruction::FMul:
- return InstDesc(Kind == RK_FloatMult, I, UAI);
+ return InstDesc(Kind == RecurKind::FMul, I, UAI);
case Instruction::FSub:
case Instruction::FAdd:
- return InstDesc(Kind == RK_FloatAdd, I, UAI);
+ return InstDesc(Kind == RecurKind::FAdd, I, UAI);
case Instruction::Select:
- if (Kind == RK_FloatAdd || Kind == RK_FloatMult)
+ if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul)
return isConditionalRdxPattern(Kind, I);
LLVM_FALLTHROUGH;
case Instruction::FCmp:
case Instruction::ICmp:
- if (Kind != RK_IntegerMinMax &&
- (!HasFunNoNaNAttr || Kind != RK_FloatMinMax))
+ if (!isIntMinMaxRecurrenceKind(Kind) &&
+ (!HasFunNoNaNAttr || !isFPMinMaxRecurrenceKind(Kind)))
return InstDesc(false, I);
return isMinMaxSelectCmpPattern(I, Prev);
}
@@ -609,50 +616,69 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
bool HasFunNoNaNAttr =
F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true";
- if (AddReductionVar(Phi, RK_IntegerAdd, TheLoop, HasFunNoNaNAttr, RedDes, DB,
+ if (AddReductionVar(Phi, RecurKind::Add, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_IntegerMult, TheLoop, HasFunNoNaNAttr, RedDes, DB,
+ if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_IntegerOr, TheLoop, HasFunNoNaNAttr, RedDes, DB,
+ if (AddReductionVar(Phi, RecurKind::Or, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an OR reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_IntegerAnd, TheLoop, HasFunNoNaNAttr, RedDes, DB,
+ if (AddReductionVar(Phi, RecurKind::And, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an AND reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_IntegerXor, TheLoop, HasFunNoNaNAttr, RedDes, DB,
+ if (AddReductionVar(Phi, RecurKind::Xor, TheLoop, HasFunNoNaNAttr, RedDes, DB,
AC, DT)) {
LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_IntegerMinMax, TheLoop, HasFunNoNaNAttr, RedDes,
+ if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, HasFunNoNaNAttr, RedDes,
DB, AC, DT)) {
- LLVM_DEBUG(dbgs() << "Found a MINMAX reduction PHI." << *Phi << "\n");
+ LLVM_DEBUG(dbgs() << "Found a SMAX reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_FloatMult, TheLoop, HasFunNoNaNAttr, RedDes, DB,
- AC, DT)) {
+ if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, HasFunNoNaNAttr, RedDes,
+ DB, AC, DT)) {
+ LLVM_DEBUG(dbgs() << "Found a SMIN reduction PHI." << *Phi << "\n");
+ return true;
+ }
+ if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, HasFunNoNaNAttr, RedDes,
+ DB, AC, DT)) {
+ LLVM_DEBUG(dbgs() << "Found a UMAX reduction PHI." << *Phi << "\n");
+ return true;
+ }
+ if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, HasFunNoNaNAttr, RedDes,
+ DB, AC, DT)) {
+ LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
+ return true;
+ }
+ if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, HasFunNoNaNAttr, RedDes,
+ DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_FloatAdd, TheLoop, HasFunNoNaNAttr, RedDes, DB,
- AC, DT)) {
+ if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, HasFunNoNaNAttr, RedDes,
+ DB, AC, DT)) {
LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RK_FloatMinMax, TheLoop, HasFunNoNaNAttr, RedDes, DB,
- AC, DT)) {
- LLVM_DEBUG(dbgs() << "Found an float MINMAX reduction PHI." << *Phi
- << "\n");
+ if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, HasFunNoNaNAttr, RedDes,
+ DB, AC, DT)) {
+ LLVM_DEBUG(dbgs() << "Found a float MAX reduction PHI." << *Phi << "\n");
+ return true;
+ }
+ if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, HasFunNoNaNAttr, RedDes,
+ DB, AC, DT)) {
+ LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
return true;
}
// Not a reduction of known type.
@@ -736,72 +762,68 @@ bool RecurrenceDescriptor::isFirstOrderRecurrence(
/// This function returns the identity element (or neutral element) for
/// the operation K.
-Constant *RecurrenceDescriptor::getRecurrenceIdentity(RecurrenceKind K,
- MinMaxRecurrenceKind MK,
- Type *Tp) {
+Constant *RecurrenceDescriptor::getRecurrenceIdentity(RecurKind K, Type *Tp) {
switch (K) {
- case RK_IntegerXor:
- case RK_IntegerAdd:
- case RK_IntegerOr:
+ case RecurKind::Xor:
+ case RecurKind::Add:
+ case RecurKind::Or:
// Adding, Xoring, Oring zero to a number does not change it.
return ConstantInt::get(Tp, 0);
- case RK_IntegerMult:
+ case RecurKind::Mul:
// Multiplying a number by 1 does not change it.
return ConstantInt::get(Tp, 1);
- case RK_IntegerAnd:
+ case RecurKind::And:
// AND-ing a number with an all-1 value does not change it.
return ConstantInt::get(Tp, -1, true);
- case RK_FloatMult:
+ case RecurKind::FMul:
// Multiplying a number by 1 does not change it.
return ConstantFP::get(Tp, 1.0L);
- case RK_FloatAdd:
+ case RecurKind::FAdd:
// Adding zero to a number does not change it.
return ConstantFP::get(Tp, 0.0L);
- case RK_IntegerMinMax:
- case RK_FloatMinMax:
- switch (MK) {
- case MRK_UIntMin:
- return ConstantInt::get(Tp, -1);
- case MRK_UIntMax:
- return ConstantInt::get(Tp, 0);
- case MRK_SIntMin:
- return ConstantInt::get(
- Tp, APInt::getSignedMaxValue(Tp->getIntegerBitWidth()));
- case MRK_SIntMax:
- return ConstantInt::get(
- Tp, APInt::getSignedMinValue(Tp->getIntegerBitWidth()));
- case MRK_FloatMin:
- return ConstantFP::getInfinity(Tp, true);
- case MRK_FloatMax:
- return ConstantFP::getInfinity(Tp, false);
- default:
- llvm_unreachable("Unknown recurrence kind");
- }
+ case RecurKind::UMin:
+ return ConstantInt::get(Tp, -1);
+ case RecurKind::UMax:
+ return ConstantInt::get(Tp, 0);
+ case RecurKind::SMin:
+ return ConstantInt::get(Tp,
+ APInt::getSignedMaxValue(Tp->getIntegerBitWidth()));
+ case RecurKind::SMax:
+ return ConstantInt::get(Tp,
+ APInt::getSignedMinValue(Tp->getIntegerBitWidth()));
+ case RecurKind::FMin:
+ return ConstantFP::getInfinity(Tp, true);
+ case RecurKind::FMax:
+ return ConstantFP::getInfinity(Tp, false);
default:
llvm_unreachable("Unknown recurrence kind");
}
}
/// This function translates the recurrence kind to an LLVM binary operator.
-unsigned RecurrenceDescriptor::getRecurrenceBinOp(RecurrenceKind Kind) {
+unsigned RecurrenceDescriptor::getRecurrenceBinOp(RecurKind Kind) {
switch (Kind) {
- case RK_IntegerAdd:
+ case RecurKind::Add:
return Instruction::Add;
- case RK_IntegerMult:
+ case RecurKind::Mul:
return Instruction::Mul;
- case RK_IntegerOr:
+ case RecurKind::Or:
return Instruction::Or;
- case RK_IntegerAnd:
+ case RecurKind::And:
return Instruction::And;
- case RK_IntegerXor:
+ case RecurKind::Xor:
return Instruction::Xor;
- case RK_FloatMult:
+ case RecurKind::FMul:
return Instruction::FMul;
- case RK_FloatAdd:
+ case RecurKind::FAdd:
return Instruction::FAdd;
- case RK_IntegerMinMax:
+ case RecurKind::SMax:
+ case RecurKind::SMin:
+ case RecurKind::UMax:
+ case RecurKind::UMin:
return Instruction::ICmp;
- case RK_FloatMinMax:
+ case RecurKind::FMax:
+ case RecurKind::FMin:
return Instruction::FCmp;
default:
llvm_unreachable("Unknown recurrence operation");
diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp
index 184eae51b2ce..a4c9f02dc64d 100644
--- a/llvm/lib/CodeGen/ExpandReductions.cpp
+++ b/llvm/lib/CodeGen/ExpandReductions.cpp
@@ -57,22 +57,22 @@ unsigned getOpcode(Intrinsic::ID ID) {
}
}
-RecurrenceDescriptor::MinMaxRecurrenceKind getMRK(Intrinsic::ID ID) {
+RecurKind getRK(Intrinsic::ID ID) {
switch (ID) {
case Intrinsic::vector_reduce_smax:
- return RecurrenceDescriptor::MRK_SIntMax;
+ return RecurKind::SMax;
case Intrinsic::vector_reduce_smin:
- return RecurrenceDescriptor::MRK_SIntMin;
+ return RecurKind::SMin;
case Intrinsic::vector_reduce_umax:
- return RecurrenceDescriptor::MRK_UIntMax;
+ return RecurKind::UMax;
case Intrinsic::vector_reduce_umin:
- return RecurrenceDescriptor::MRK_UIntMin;
+ return RecurKind::UMin;
case Intrinsic::vector_reduce_fmax:
- return RecurrenceDescriptor::MRK_FloatMax;
+ return RecurKind::FMax;
case Intrinsic::vector_reduce_fmin:
- return RecurrenceDescriptor::MRK_FloatMin;
+ return RecurKind::FMin;
default:
- return RecurrenceDescriptor::MRK_Invalid;
+ return RecurKind::None;
}
}
@@ -108,7 +108,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
FastMathFlags FMF =
isa<FPMathOperator>(II) ? II->getFastMathFlags() : FastMathFlags{};
Intrinsic::ID ID = II->getIntrinsicID();
- RecurrenceDescriptor::MinMaxRecurrenceKind MRK = getMRK(ID);
+ RecurKind RK = getRK(ID);
Value *Rdx = nullptr;
IRBuilder<> Builder(II);
@@ -123,13 +123,13 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
Value *Acc = II->getArgOperand(0);
Value *Vec = II->getArgOperand(1);
if (!FMF.allowReassoc())
- Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK);
+ Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), RK);
else {
if (!isPowerOf2_32(
cast<FixedVectorType>(Vec->getType())->getNumElements()))
continue;
- Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
+ Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
Rdx = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(ID),
Acc, Rdx, "bin.rdx");
}
@@ -149,7 +149,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
cast<FixedVectorType>(Vec->getType())->getNumElements()))
continue;
- Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
+ Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
break;
}
case Intrinsic::vector_reduce_fmax:
@@ -163,7 +163,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
!FMF.isFast())
continue;
- Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
+ Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
break;
}
}
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 8dc7709c6e55..96f1d4219bac 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -868,29 +868,28 @@ bool llvm::hasIterationCountInvariantInParent(Loop *InnerLoop,
return true;
}
-Value *llvm::createMinMaxOp(IRBuilderBase &Builder,
- RecurrenceDescriptor::MinMaxRecurrenceKind RK,
- Value *Left, Value *Right) {
+Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
+ Value *Right) {
CmpInst::Predicate P = CmpInst::ICMP_NE;
switch (RK) {
default:
llvm_unreachable("Unknown min/max recurrence kind");
- case RecurrenceDescriptor::MRK_UIntMin:
+ case RecurKind::UMin:
P = CmpInst::ICMP_ULT;
break;
- case RecurrenceDescriptor::MRK_UIntMax:
+ case RecurKind::UMax:
P = CmpInst::ICMP_UGT;
break;
- case RecurrenceDescriptor::MRK_SIntMin:
+ case RecurKind::SMin:
P = CmpInst::ICMP_SLT;
break;
- case RecurrenceDescriptor::MRK_SIntMax:
+ case RecurKind::SMax:
P = CmpInst::ICMP_SGT;
break;
- case RecurrenceDescriptor::MRK_FloatMin:
+ case RecurKind::FMin:
P = CmpInst::FCMP_OLT;
break;
- case RecurrenceDescriptor::MRK_FloatMax:
+ case RecurKind::FMax:
P = CmpInst::FCMP_OGT;
break;
}
@@ -907,11 +906,9 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder,
}
// Helper to generate an ordered reduction.
-Value *
-llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
- unsigned Op,
- RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
- ArrayRef<Value *> RedOps) {
+Value *llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
+ unsigned Op, RecurKind RdxKind,
+ ArrayRef<Value *> RedOps) {
unsigned VF = cast<FixedVectorType>(Src->getType())->getNumElements();
// Extract and apply reduction ops in ascending order:
@@ -925,9 +922,9 @@ llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
Result = Builder.CreateBinOp((Instruction::BinaryOps)Op, Result, Ext,
"bin.rdx");
} else {
- assert(MinMaxKind != RecurrenceDescriptor::MRK_Invalid &&
+ assert(RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind) &&
"Invalid min/max");
- Result = createMinMaxOp(Builder, MinMaxKind, Result, Ext);
+ Result = createMinMaxOp(Builder, RdxKind, Result, Ext);
}
if (!RedOps.empty())
@@ -938,10 +935,9 @@ llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
}
// Helper to generate a log2 shuffle reduction.
-Value *
-llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
- RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
- ArrayRef<Value *> RedOps) {
+Value *llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src,
+ unsigned Op, RecurKind RdxKind,
+ ArrayRef<Value *> RedOps) {
unsigned VF = cast<FixedVectorType>(Src->getType())->getNumElements();
// VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
// and vector ops, reducing the set of values being computed by half each
@@ -965,9 +961,9 @@ llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
TmpVec = Builder.CreateBinOp((Instruction::BinaryOps)Op, TmpVec, Shuf,
"bin.rdx");
} else {
- assert(MinMaxKind != RecurrenceDescriptor::MRK_Invalid &&
+ assert(RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind) &&
"Invalid min/max");
- TmpVec = createMinMaxOp(Builder, MinMaxKind, TmpVec, Shuf);
+ TmpVec = createMinMaxOp(Builder, RdxKind, TmpVec, Shuf);
}
if (!RedOps.empty())
propagateIRFlags(TmpVec, RedOps);
@@ -981,16 +977,14 @@ llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
}
-/// Create a simple vector reduction specified by an opcode and some
-/// flags (if generating min/max reductions).
-Value *llvm::createSimpleTargetReduction(
- IRBuilderBase &Builder, const TargetTransformInfo *TTI, unsigned Opcode,
- Value *Src, RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind,
- ArrayRef<Value *> RedOps) {
+Value *llvm::createSimpleTargetReduction(IRBuilderBase &Builder,
+ const TargetTransformInfo *TTI,
+ unsigned Opcode, Value *Src,
+ RecurKind RdxKind,
+ ArrayRef<Value *> RedOps) {
auto *SrcVTy = cast<VectorType>(Src->getType());
std::function<Value *()> BuildFunc;
- using RD = RecurrenceDescriptor;
switch (Opcode) {
case Instruction::Add:
BuildFunc = [&]() { return Builder.CreateAddReduce(Src); };
@@ -1022,17 +1016,17 @@ Value *llvm::createSimpleTargetReduction(
};
break;
case Instruction::ICmp:
- switch (MinMaxKind) {
- case RD::MRK_SIntMax:
+ switch (RdxKind) {
+ case RecurKind::SMax:
BuildFunc = [&]() { return Builder.CreateIntMaxReduce(Src, true); };
break;
- case RD::MRK_SIntMin:
+ case RecurKind::SMin:
BuildFunc = [&]() { return Builder.CreateIntMinReduce(Src, true); };
break;
- case RD::MRK_UIntMax:
+ case RecurKind::UMax:
BuildFunc = [&]() { return Builder.CreateIntMaxReduce(Src, false); };
break;
- case RD::MRK_UIntMin:
+ case RecurKind::UMin:
BuildFunc = [&]() { return Builder.CreateIntMinReduce(Src, false); };
break;
default:
@@ -1040,9 +1034,9 @@ Value *llvm::createSimpleTargetReduction(
}
break;
case Instruction::FCmp:
- assert((MinMaxKind == RD::MRK_FloatMax || MinMaxKind == RD::MRK_FloatMin) &&
+ assert((RdxKind == RecurKind::FMax || RdxKind == RecurKind::FMin) &&
"Unexpected min/max reduction type");
- if (MinMaxKind == RD::MRK_FloatMax)
+ if (RdxKind == RecurKind::FMax)
BuildFunc = [&]() { return Builder.CreateFPMaxReduce(Src); };
else
BuildFunc = [&]() { return Builder.CreateFPMinReduce(Src); };
@@ -1051,32 +1045,26 @@ Value *llvm::createSimpleTargetReduction(
llvm_unreachable("Unhandled opcode");
}
TargetTransformInfo::ReductionFlags RdxFlags;
- RdxFlags.IsMaxOp = MinMaxKind == RD::MRK_SIntMax ||
- MinMaxKind == RD::MRK_UIntMax ||
- MinMaxKind == RD::MRK_FloatMax;
- RdxFlags.IsSigned =
- MinMaxKind == RD::MRK_SIntMax || MinMaxKind == RD::MRK_SIntMin;
+ RdxFlags.IsMaxOp = RdxKind == RecurKind::SMax ||
+ RdxKind == RecurKind::UMax ||
+ RdxKind == RecurKind::FMax;
+ RdxFlags.IsSigned = RdxKind == RecurKind::SMax || RdxKind == RecurKind::SMin;
if (ForceReductionIntrinsic ||
TTI->useReductionIntrinsic(Opcode, Src->getType(), RdxFlags))
return BuildFunc();
- return getShuffleReduction(Builder, Src, Opcode, MinMaxKind, RedOps);
+ return getShuffleReduction(Builder, Src, Opcode, RdxKind, RedOps);
}
-/// Create a vector reduction using a given recurrence descriptor.
Value *llvm::createTargetReduction(IRBuilderBase &B,
const TargetTransformInfo *TTI,
RecurrenceDescriptor &Desc, Value *Src) {
// TODO: Support in-order reductions based on the recurrence descriptor.
- using RD = RecurrenceDescriptor;
-
// All ops in the reduction inherit fast-math-flags from the recurrence
// descriptor.
IRBuilderBase::FastMathFlagGuard FMFGuard(B);
B.setFastMathFlags(Desc.getFastMathFlags());
-
- RD::MinMaxRecurrenceKind MMKind = Desc.getMinMaxRecurrenceKind();
return createSimpleTargetReduction(B, TTI, Desc.getRecurrenceBinOp(), Src,
- MMKind);
+ Desc.getRecurrenceKind());
}
void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3bc946603eb0..472251840463 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4152,11 +4152,9 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
"Unable to find the reduction variable");
RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];
- RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
+ RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
- RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =
- RdxDesc.getMinMaxRecurrenceKind();
setDebugLocFromInst(Builder, ReductionStartValue);
bool IsInLoopReductionPhi = Cost->isInLoopReduction(Phi);
@@ -4173,8 +4171,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
// one for multiplication, -1 for And.
Value *Identity;
Value *VectorStart;
- if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||
- RK == RecurrenceDescriptor::RK_FloatMinMax) {
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
// MinMax reduction have the start value as their identify.
if (VF.isScalar() || IsInLoopReductionPhi) {
VectorStart = Identity = ReductionStartValue;
@@ -4185,7 +4182,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
} else {
// Handle other reduction kinds:
Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
- RK, MinMaxKind, VecTy->getScalarType());
+ RK, VecTy->getScalarType());
if (VF.isScalar() || IsInLoopReductionPhi) {
Identity = Iden;
// This vector is the Identity vector where the first element is the
@@ -4318,8 +4315,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
ReducedPartRdx, "bin.rdx"),
RdxDesc.getFastMathFlags());
else
- ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
- RdxPart);
+ ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
}
// Create the reduction after the loop. Note that inloop reductions create the
@@ -4372,9 +4368,8 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
void InnerLoopVectorizer::clearReductionWrapFlags(
RecurrenceDescriptor &RdxDesc) {
- RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
- if (RK != RecurrenceDescriptor::RK_IntegerAdd &&
- RK != RecurrenceDescriptor::RK_IntegerMult)
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ if (RK != RecurKind::Add && RK != RecurKind::Mul)
return;
Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr();
@@ -8395,8 +8390,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
}
for (auto &Reduction : CM.getInLoopReductionChains()) {
PHINode *Phi = Reduction.first;
- RecurrenceDescriptor::RecurrenceKind Kind =
- Legal->getReductionVars()[Phi].getRecurrenceKind();
+ RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind();
const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;
RecipeBuilder.recordRecipeOf(Phi);
@@ -8404,10 +8398,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
RecipeBuilder.recordRecipeOf(R);
// For min/max reducitons, where we have a pair of icmp/select, we also
// need to record the ICmp recipe, so it can be removed later.
- if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
- Kind == RecurrenceDescriptor::RK_FloatMinMax) {
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
RecipeBuilder.recordRecipeOf(cast<Instruction>(R->getOperand(0)));
- }
}
}
@@ -8621,12 +8613,11 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
Instruction *Chain = Phi;
for (Instruction *R : ReductionOperations) {
VPRecipeBase *WidenRecipe = RecipeBuilder.getRecipe(R);
- RecurrenceDescriptor::RecurrenceKind Kind = RdxDesc.getRecurrenceKind();
+ RecurKind Kind = RdxDesc.getRecurrenceKind();
VPValue *ChainOp = Plan->getVPValue(Chain);
unsigned FirstOpId;
- if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
- Kind == RecurrenceDescriptor::RK_FloatMinMax) {
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
assert(isa<VPWidenSelectRecipe>(WidenRecipe) &&
"Expected to replace a VPWidenSelectSC");
FirstOpId = 1;
@@ -8651,8 +8642,7 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe);
WidenRecipe->eraseFromParent();
- if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
- Kind == RecurrenceDescriptor::RK_FloatMinMax) {
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
VPRecipeBase *CompareRecipe =
RecipeBuilder.getRecipe(cast<Instruction>(R->getOperand(0)));
assert(isa<VPWidenRecipe>(CompareRecipe) &&
@@ -8769,13 +8759,13 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
void VPReductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Reduction being replicated.");
for (unsigned Part = 0; Part < State.UF; ++Part) {
- RecurrenceDescriptor::RecurrenceKind Kind = RdxDesc->getRecurrenceKind();
+ RecurKind Kind = RdxDesc->getRecurrenceKind();
Value *NewVecOp = State.get(getVecOp(), Part);
if (VPValue *Cond = getCondOp()) {
Value *NewCond = State.get(Cond, Part);
VectorType *VecTy = cast<VectorType>(NewVecOp->getType());
Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
- Kind, RdxDesc->getMinMaxRecurrenceKind(), VecTy->getElementType());
+ Kind, VecTy->getElementType());
Constant *IdenVec =
ConstantVector::getSplat(VecTy->getElementCount(), Iden);
Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, IdenVec);
@@ -8785,10 +8775,9 @@ void VPReductionRecipe::execute(VPTransformState &State) {
createTargetReduction(State.Builder, TTI, *RdxDesc, NewVecOp);
Value *PrevInChain = State.get(getChainOp(), Part);
Value *NextInChain;
- if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
- Kind == RecurrenceDescriptor::RK_FloatMinMax) {
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
NextInChain =
- createMinMaxOp(State.Builder, RdxDesc->getMinMaxRecurrenceKind(),
+ createMinMaxOp(State.Builder, RdxDesc->getRecurrenceKind(),
NewRed, PrevInChain);
} else {
NextInChain = State.Builder.CreateBinOp(
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2bbac1ac549e..2de09089be0c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6446,7 +6446,6 @@ class HorizontalReduction {
SmallVector<Value *, 32> ReducedVals;
// Use map vector to make stable output.
MapVector<Instruction *, Value *> ExtraArgs;
- using RD = RecurrenceDescriptor;
/// Contains info about operation, like its opcode, left and right operands.
class OperationData {
@@ -6454,27 +6453,29 @@ class HorizontalReduction {
unsigned Opcode = 0;
/// Kind of the reduction operation.
- RD::RecurrenceKind Kind = RD::RK_NoRecurrence;
- RD::MinMaxRecurrenceKind MMKind = RD::MRK_Invalid;
+ RecurKind Kind = RecurKind::None;
/// Checks if the reduction operation can be vectorized.
bool isVectorizable() const {
switch (Kind) {
- case RD::RK_IntegerAdd:
+ case RecurKind::Add:
return Opcode == Instruction::Add;
- case RD::RK_IntegerMult:
+ case RecurKind::Mul:
return Opcode == Instruction::Mul;
- case RD::RK_IntegerOr:
+ case RecurKind::Or:
return Opcode == Instruction::Or;
- case RD::RK_IntegerAnd:
+ case RecurKind::And:
return Opcode == Instruction::And;
- case RD::RK_IntegerXor:
+ case RecurKind::Xor:
return Opcode == Instruction::Xor;
- case RD::RK_FloatAdd:
+ case RecurKind::FAdd:
return Opcode == Instruction::FAdd;
- case RD::RK_FloatMult:
+ case RecurKind::FMul:
return Opcode == Instruction::FMul;
- case RD::RK_IntegerMinMax:
+ case RecurKind::SMax:
+ case RecurKind::SMin:
+ case RecurKind::UMax:
+ case RecurKind::UMin:
return Opcode == Instruction::ICmp;
default:
return false;
@@ -6486,27 +6487,34 @@ class HorizontalReduction {
const Twine &Name) const {
assert(isVectorizable() && "Unhandled reduction operation.");
switch (Kind) {
- case RD::RK_IntegerAdd:
- case RD::RK_IntegerMult:
- case RD::RK_IntegerOr:
- case RD::RK_IntegerAnd:
- case RD::RK_IntegerXor:
- case RD::RK_FloatAdd:
- case RD::RK_FloatMult:
+ case RecurKind::Add:
+ case RecurKind::Mul:
+ case RecurKind::Or:
+ case RecurKind::And:
+ case RecurKind::Xor:
+ case RecurKind::FAdd:
+ case RecurKind::FMul:
return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
Name);
- case RD::RK_IntegerMinMax: {
+ case RecurKind::SMax: {
assert(Opcode == Instruction::ICmp && "Expected integer types.");
- ICmpInst::Predicate Pred;
- switch (MMKind) {
- case RD::MRK_SIntMax: Pred = ICmpInst::ICMP_SGT; break;
- case RD::MRK_SIntMin: Pred = ICmpInst::ICMP_SLT; break;
- case RD::MRK_UIntMax: Pred = ICmpInst::ICMP_UGT; break;
- case RD::MRK_UIntMin: Pred = ICmpInst::ICMP_ULT; break;
- default: llvm_unreachable("Unexpected min/max value");
- }
- Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS, Name);
+ Value *Cmp = Builder.CreateICmpSGT(LHS, RHS, Name);
+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
+ }
+ case RecurKind::SMin: {
+ assert(Opcode == Instruction::ICmp && "Expected integer types.");
+ Value *Cmp = Builder.CreateICmpSLT(LHS, RHS, Name);
+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
+ }
+ case RecurKind::UMax: {
+ assert(Opcode == Instruction::ICmp && "Expected integer types.");
+ Value *Cmp = Builder.CreateICmpUGT(LHS, RHS, Name);
+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
+ }
+ case RecurKind::UMin: {
+ assert(Opcode == Instruction::ICmp && "Expected integer types.");
+ Value *Cmp = Builder.CreateICmpULT(LHS, RHS, Name);
return Builder.CreateSelect(Cmp, LHS, RHS, Name);
}
default:
@@ -6524,32 +6532,31 @@ class HorizontalReduction {
}
/// Constructor for reduction operations with opcode and type.
- OperationData(unsigned Opcode, RD::RecurrenceKind RdxKind,
- RD::MinMaxRecurrenceKind MinMaxKind)
- : Opcode(Opcode), Kind(RdxKind), MMKind(MinMaxKind) {
- assert(Kind != RD::RK_NoRecurrence && "Expected reduction operation.");
+ OperationData(unsigned Opcode, RecurKind RdxKind)
+ : Opcode(Opcode), Kind(RdxKind) {
+ assert(Kind != RecurKind::None && "Expected reduction operation.");
}
explicit operator bool() const { return Opcode; }
/// Return true if this operation is any kind of minimum or maximum.
bool isMinMax() const {
- assert(Kind != RD::RK_NoRecurrence && "Expected reduction operation.");
- return Kind == RD::RK_IntegerMinMax;
+ assert(Kind != RecurKind::None && "Expected reduction operation.");
+ return RecurrenceDescriptor::isIntMinMaxRecurrenceKind(Kind);
}
/// Get the index of the first operand.
unsigned getFirstOperandIndex() const {
assert(!!*this && "The opcode is not set.");
// We allow calling this before 'Kind' is set, so handle that specially.
- if (Kind == RD::RK_NoRecurrence)
+ if (Kind == RecurKind::None)
return 0;
return isMinMax() ? 1 : 0;
}
/// Total number of operands in the reduction operation.
unsigned getNumberOfOperands() const {
- assert(Kind != RD::RK_NoRecurrence && !!*this &&
+ assert(Kind != RecurKind::None && !!*this &&
"Expected reduction operation.");
return isMinMax() ? 3 : 2;
}
@@ -6557,7 +6564,7 @@ class HorizontalReduction {
/// Checks if the instruction is in basic block \p BB.
/// For a min/max reduction check that both compare and select are in \p BB.
bool hasSameParent(Instruction *I, BasicBlock *BB, bool IsRedOp) const {
- assert(Kind != RD::RK_NoRecurrence && !!*this &&
+ assert(Kind != RecurKind::None && !!*this &&
"Expected reduction operation.");
if (IsRedOp && isMinMax()) {
auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
@@ -6568,7 +6575,7 @@ class HorizontalReduction {
/// Expected number of uses for reduction operations/reduced values.
bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
- assert(Kind != RD::RK_NoRecurrence && !!*this &&
+ assert(Kind != RecurKind::None && !!*this &&
"Expected reduction operation.");
// SelectInst must be used twice while the condition op must have single
// use only.
@@ -6583,7 +6590,7 @@ class HorizontalReduction {
/// Initializes the list of reduction operations.
void initReductionOps(ReductionOpsListType &ReductionOps) {
- assert(Kind != RD::RK_NoRecurrence && !!*this &&
+ assert(Kind != RecurKind::None && !!*this &&
"Expected reduction operation.");
if (isMinMax())
ReductionOps.assign(2, ReductionOpsType());
@@ -6593,7 +6600,7 @@ class HorizontalReduction {
/// Add all reduction operations for the reduction instruction \p I.
void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
- assert(Kind != RD::RK_NoRecurrence && !!*this &&
+ assert(Kind != RecurKind::None && !!*this &&
"Expected reduction operation.");
if (isMinMax()) {
ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
@@ -6605,16 +6612,14 @@ class HorizontalReduction {
/// Checks if instruction is associative and can be vectorized.
bool isAssociative(Instruction *I) const {
- assert(Kind != RD::RK_NoRecurrence && *this &&
+ assert(Kind != RecurKind::None && *this &&
"Expected reduction operation.");
- switch (Kind) {
- case RD::RK_IntegerMinMax:
+ if (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(Kind)) {
assert(Opcode == Instruction::ICmp &&
"Only integer compare operation is expected.");
return true;
- default:
- return I->isAssociative();
}
+ return I->isAssociative();
}
/// Checks if the reduction operation can be vectorized.
@@ -6632,7 +6637,7 @@ class HorizontalReduction {
bool operator!=(const OperationData &OD) const { return !(*this == OD); }
void clear() {
Opcode = 0;
- Kind = RD::RK_NoRecurrence;
+ Kind = RecurKind::None;
}
/// Get the opcode of the reduction operation.
@@ -6642,15 +6647,14 @@ class HorizontalReduction {
}
/// Get kind of reduction data.
- RD::RecurrenceKind getKind() const { return Kind; }
- RD::MinMaxRecurrenceKind getMinMaxKind() const { return MMKind; }
+ RecurKind getKind() const { return Kind; }
Value *getLHS(Instruction *I) const {
- if (Kind == RD::RK_NoRecurrence)
+ if (Kind == RecurKind::None)
return nullptr;
return I->getOperand(getFirstOperandIndex());
}
Value *getRHS(Instruction *I) const {
- if (Kind == RD::RK_NoRecurrence)
+ if (Kind == RecurKind::None)
return nullptr;
return I->getOperand(getFirstOperandIndex() + 1);
}
@@ -6664,16 +6668,19 @@ class HorizontalReduction {
"Expected add|fadd or min/max reduction operation.");
auto *Op = createOp(Builder, LHS, RHS, Name);
switch (Kind) {
- case RD::RK_IntegerAdd:
- case RD::RK_IntegerMult:
- case RD::RK_IntegerOr:
- case RD::RK_IntegerAnd:
- case RD::RK_IntegerXor:
- case RD::RK_FloatAdd:
- case RD::RK_FloatMult:
+ case RecurKind::Add:
+ case RecurKind::Mul:
+ case RecurKind::Or:
+ case RecurKind::And:
+ case RecurKind::Xor:
+ case RecurKind::FAdd:
+ case RecurKind::FMul:
propagateIRFlags(Op, ReductionOps[0]);
return Op;
- case RD::RK_IntegerMinMax:
+ case RecurKind::SMax:
+ case RecurKind::SMin:
+ case RecurKind::UMax:
+ case RecurKind::UMin:
if (auto *SI = dyn_cast<SelectInst>(Op))
propagateIRFlags(SI->getCondition(), ReductionOps[0]);
propagateIRFlags(Op, ReductionOps[1]);
@@ -6690,16 +6697,19 @@ class HorizontalReduction {
"Expected add|fadd or min/max reduction operation.");
auto *Op = createOp(Builder, LHS, RHS, Name);
switch (Kind) {
- case RD::RK_IntegerAdd:
- case RD::RK_IntegerMult:
- case RD::RK_IntegerOr:
- case RD::RK_IntegerAnd:
- case RD::RK_IntegerXor:
- case RD::RK_FloatAdd:
- case RD::RK_FloatMult:
+ case RecurKind::Add:
+ case RecurKind::Mul:
+ case RecurKind::Or:
+ case RecurKind::And:
+ case RecurKind::Xor:
+ case RecurKind::FAdd:
+ case RecurKind::FMul:
propagateIRFlags(Op, I);
return Op;
- case RD::RK_IntegerMinMax:
+ case RecurKind::SMax:
+ case RecurKind::SMin:
+ case RecurKind::UMax:
+ case RecurKind::UMin:
if (auto *SI = dyn_cast<SelectInst>(Op)) {
propagateIRFlags(SI->getCondition(),
cast<SelectInst>(I)->getCondition());
@@ -6751,32 +6761,28 @@ class HorizontalReduction {
TargetTransformInfo::ReductionFlags RdxFlags;
if (match(I, m_Add(m_Value(), m_Value())))
- return OperationData(I->getOpcode(), RD::RK_IntegerAdd, RD::MRK_Invalid);
+ return OperationData(I->getOpcode(), RecurKind::Add);
if (match(I, m_Mul(m_Value(), m_Value())))
- return OperationData(I->getOpcode(), RD::RK_IntegerMult, RD::MRK_Invalid);
+ return OperationData(I->getOpcode(), RecurKind::Mul);
if (match(I, m_And(m_Value(), m_Value())))
- return OperationData(I->getOpcode(), RD::RK_IntegerAnd, RD::MRK_Invalid);
+ return OperationData(I->getOpcode(), RecurKind::And);
if (match(I, m_Or(m_Value(), m_Value())))
- return OperationData(I->getOpcode(), RD::RK_IntegerOr, RD::MRK_Invalid);
+ return OperationData(I->getOpcode(), RecurKind::Or);
if (match(I, m_Xor(m_Value(), m_Value())))
- return OperationData(I->getOpcode(), RD::RK_IntegerXor, RD::MRK_Invalid);
+ return OperationData(I->getOpcode(), RecurKind::Xor);
if (match(I, m_FAdd(m_Value(), m_Value())))
- return OperationData(I->getOpcode(), RD::RK_FloatAdd, RD::MRK_Invalid);
+ return OperationData(I->getOpcode(), RecurKind::FAdd);
if (match(I, m_FMul(m_Value(), m_Value())))
- return OperationData(I->getOpcode(), RD::RK_FloatMult, RD::MRK_Invalid);
+ return OperationData(I->getOpcode(), RecurKind::FMul);
if (match(I, m_SMax(m_Value(), m_Value())))
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_SIntMax);
+ return OperationData(Instruction::ICmp, RecurKind::SMax);
if (match(I, m_SMin(m_Value(), m_Value())))
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_SIntMin);
+ return OperationData(Instruction::ICmp, RecurKind::SMin);
if (match(I, m_UMax(m_Value(), m_Value())))
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_UIntMax);
+ return OperationData(Instruction::ICmp, RecurKind::UMax);
if (match(I, m_UMin(m_Value(), m_Value())))
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_UIntMin);
+ return OperationData(Instruction::ICmp, RecurKind::UMin);
if (auto *Select = dyn_cast<SelectInst>(I)) {
// Try harder: look for min/max pattern based on instructions producing
@@ -6822,20 +6828,16 @@ class HorizontalReduction {
return OperationData(*I);
case CmpInst::ICMP_SGT:
case CmpInst::ICMP_SGE:
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_SIntMax);
+ return OperationData(Instruction::ICmp, RecurKind::SMax);
case CmpInst::ICMP_SLT:
case CmpInst::ICMP_SLE:
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_SIntMin);
+ return OperationData(Instruction::ICmp, RecurKind::SMin);
case CmpInst::ICMP_UGT:
case CmpInst::ICMP_UGE:
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_UIntMax);
+ return OperationData(Instruction::ICmp, RecurKind::UMax);
case CmpInst::ICMP_ULT:
case CmpInst::ICMP_ULE:
- return OperationData(Instruction::ICmp, RD::RK_IntegerMinMax,
- RD::MRK_UIntMin);
+ return OperationData(Instruction::ICmp, RecurKind::UMin);
}
}
return OperationData(*I);
@@ -7171,13 +7173,13 @@ class HorizontalReduction {
int PairwiseRdxCost;
int SplittingRdxCost;
switch (RdxTreeInst.getKind()) {
- case RD::RK_IntegerAdd:
- case RD::RK_IntegerMult:
- case RD::RK_IntegerOr:
- case RD::RK_IntegerAnd:
- case RD::RK_IntegerXor:
- case RD::RK_FloatAdd:
- case RD::RK_FloatMult:
+ case RecurKind::Add:
+ case RecurKind::Mul:
+ case RecurKind::Or:
+ case RecurKind::And:
+ case RecurKind::Xor:
+ case RecurKind::FAdd:
+ case RecurKind::FMul:
PairwiseRdxCost =
TTI->getArithmeticReductionCost(RdxTreeInst.getOpcode(), VecTy,
/*IsPairwiseForm=*/true);
@@ -7185,10 +7187,13 @@ class HorizontalReduction {
TTI->getArithmeticReductionCost(RdxTreeInst.getOpcode(), VecTy,
/*IsPairwiseForm=*/false);
break;
- case RD::RK_IntegerMinMax: {
+ case RecurKind::SMax:
+ case RecurKind::SMin:
+ case RecurKind::UMax:
+ case RecurKind::UMin: {
auto *VecCondTy = cast<VectorType>(CmpInst::makeCmpResultType(VecTy));
- RD::MinMaxRecurrenceKind MMKind = RdxTreeInst.getMinMaxKind();
- bool IsUnsigned = MMKind == RD::MRK_UIntMax || MMKind == RD::MRK_UIntMin;
+ RecurKind Kind = RdxTreeInst.getKind();
+ bool IsUnsigned = Kind == RecurKind::UMax || Kind == RecurKind::UMin;
PairwiseRdxCost =
TTI->getMinMaxReductionCost(VecTy, VecCondTy,
/*IsPairwiseForm=*/true, IsUnsigned);
@@ -7206,17 +7211,20 @@ class HorizontalReduction {
int ScalarReduxCost = 0;
switch (RdxTreeInst.getKind()) {
- case RD::RK_IntegerAdd:
- case RD::RK_IntegerMult:
- case RD::RK_IntegerOr:
- case RD::RK_IntegerAnd:
- case RD::RK_IntegerXor:
- case RD::RK_FloatAdd:
- case RD::RK_FloatMult:
+ case RecurKind::Add:
+ case RecurKind::Mul:
+ case RecurKind::Or:
+ case RecurKind::And:
+ case RecurKind::Xor:
+ case RecurKind::FAdd:
+ case RecurKind::FMul:
ScalarReduxCost =
TTI->getArithmeticInstrCost(RdxTreeInst.getOpcode(), ScalarTy);
break;
- case RD::RK_IntegerMinMax:
+ case RecurKind::SMax:
+ case RecurKind::SMin:
+ case RecurKind::UMax:
+ case RecurKind::UMin:
ScalarReduxCost =
TTI->getCmpSelInstrCost(RdxTreeInst.getOpcode(), ScalarTy) +
TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
@@ -7249,7 +7257,7 @@ class HorizontalReduction {
assert(Builder.getFastMathFlags().isFast() && "Expected 'fast' FMF");
return createSimpleTargetReduction(
Builder, TTI, RdxTreeInst.getOpcode(), VectorizedValue,
- RdxTreeInst.getMinMaxKind(), ReductionOps.back());
+ RdxTreeInst.getKind(), ReductionOps.back());
}
Value *TmpVec = VectorizedValue;
More information about the llvm-branch-commits
mailing list