<div dir="ltr">Looks like this caused:<div><br></div><div><pre style="color:rgb(0,0,0)"><font face="monospace, monospace">include/llvm/Transforms/Utils/LoopUtils.h:75:16: warning: private field 'PatternLastInst' is not used [-Wunused-private-field]</font></pre></div></div><div class="gmail_extra"><br><div class="gmail_quote">On Sun, Apr 19, 2015 at 9:38 PM, Karthik Bhat <span dir="ltr"><<a href="mailto:kv.bhat@samsung.com" target="_blank">kv.bhat@samsung.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: karthik<br>
Date: Sun Apr 19 23:38:33 2015<br>
New Revision: 235284<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=235284&view=rev" target="_blank">http://llvm.org/viewvc/llvm-project?rev=235284&view=rev</a><br>
Log:<br>
[NFC] Refactor identification of reductions as common utility function.<br>
This patch refactors reduction identification code out of LoopVectorizer and<br>
exposes them as common utilities.<br>
No functional change.<br>
Review: <a href="http://reviews.llvm.org/D9046" target="_blank">http://reviews.llvm.org/D9046</a><br>
<br>
<br>
Added:<br>
llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp<br>
Modified:<br>
llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h<br>
llvm/trunk/lib/Transforms/Utils/CMakeLists.txt<br>
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp<br>
<br>
Modified: llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h?rev=235284&r1=235283&r2=235284&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h?rev=235284&r1=235283&r2=235284&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h (original)<br>
+++ llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h Sun Apr 19 23:38:33 2015<br>
@@ -16,6 +16,7 @@<br>
<br>
#include "llvm/ADT/SmallVector.h"<br>
#include "llvm/IR/Dominators.h"<br>
+#include "llvm/IR/IRBuilder.h"<br>
<br>
namespace llvm {<br>
class AliasAnalysis;<br>
@@ -42,6 +43,132 @@ struct LICMSafetyInfo {<br>
{}<br>
};<br>
<br>
+/// This POD struct holds information about a potential reduction operation.<br>
+class ReductionInstDesc {<br>
+<br>
+public:<br>
+ // This enum represents the kind of minmax reduction.<br>
+ enum MinMaxReductionKind {<br>
+ MRK_Invalid,<br>
+ MRK_UIntMin,<br>
+ MRK_UIntMax,<br>
+ MRK_SIntMin,<br>
+ MRK_SIntMax,<br>
+ MRK_FloatMin,<br>
+ MRK_FloatMax<br>
+ };<br>
+ ReductionInstDesc(bool IsRedux, Instruction *I)<br>
+ : IsReduction(IsRedux), PatternLastInst(I), MinMaxKind(MRK_Invalid) {}<br>
+<br>
+ ReductionInstDesc(Instruction *I, MinMaxReductionKind K)<br>
+ : IsReduction(true), PatternLastInst(I), MinMaxKind(K) {}<br>
+<br>
+ bool isReduction() { return IsReduction; }<br>
+<br>
+ MinMaxReductionKind getMinMaxKind() { return MinMaxKind; }<br>
+<br>
+private:<br>
+ // Is this instruction a reduction candidate.<br>
+ bool IsReduction;<br>
+ // The last instruction in a min/max pattern (select of the select(icmp())<br>
+ // pattern), or the current reduction instruction otherwise.<br>
+ Instruction *PatternLastInst;<br>
+ // If this is a min/max pattern the comparison predicate.<br>
+ MinMaxReductionKind MinMaxKind;<br>
+};<br>
+<br>
+/// This struct holds information about reduction variables.<br>
+class ReductionDescriptor {<br>
+<br>
+public:<br>
+ /// This enum represents the kinds of reductions that we support.<br>
+ enum ReductionKind {<br>
+ RK_NoReduction, ///< Not a reduction.<br>
+ RK_IntegerAdd, ///< Sum of integers.<br>
+ RK_IntegerMult, ///< Product of integers.<br>
+ RK_IntegerOr, ///< Bitwise or logical OR of numbers.<br>
+ RK_IntegerAnd, ///< Bitwise or logical AND of numbers.<br>
+ RK_IntegerXor, ///< Bitwise or logical XOR of numbers.<br>
+ RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()).<br>
+ RK_FloatAdd, ///< Sum of floats.<br>
+ RK_FloatMult, ///< Product of floats.<br>
+ RK_FloatMinMax ///< Min/max implemented in terms of select(cmp()).<br>
+ };<br>
+<br>
+ ReductionDescriptor()<br>
+ : StartValue(nullptr), LoopExitInstr(nullptr), Kind(RK_NoReduction),<br>
+ MinMaxKind(ReductionInstDesc::MRK_Invalid) {}<br>
+<br>
+ ReductionDescriptor(Value *Start, Instruction *Exit, ReductionKind K,<br>
+ ReductionInstDesc::MinMaxReductionKind MK)<br>
+ : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK) {}<br>
+<br>
+ /// Returns a struct describing if the instruction 'I' can be a reduction<br>
+ /// variable of type 'Kind'. If the reduction is a min/max pattern of<br>
+ /// select(icmp()) this function advances the instruction pointer 'I' from the<br>
+ /// compare instruction to the select instruction and stores this pointer in<br>
+ /// 'PatternLastInst' member of the returned struct.<br>
+ static ReductionInstDesc isReductionInstr(Instruction *I, ReductionKind Kind,<br>
+ ReductionInstDesc &Prev,<br>
+ bool HasFunNoNaNAttr);<br>
+<br>
+ /// Returns true if instuction I has multiple uses in Insts<br>
+ static bool hasMultipleUsesOf(Instruction *I,<br>
+ SmallPtrSetImpl<Instruction *> &Insts);<br>
+<br>
+ /// Returns true if all uses of the instruction I is within the Set.<br>
+ static bool areAllUsesIn(Instruction *I, SmallPtrSetImpl<Instruction *> &Set);<br>
+<br>
+ /// Returns a struct describing if the instruction if the instruction is a<br>
+ /// Select(ICmp(X, Y), X, Y) instruction pattern corresponding to a min(X, Y)<br>
+ /// or max(X, Y).<br>
+ static ReductionInstDesc isMinMaxSelectCmpPattern(Instruction *I,<br>
+ ReductionInstDesc &Prev);<br>
+<br>
+ /// Returns identity corresponding to the ReductionKind.<br>
+ static Constant *getReductionIdentity(ReductionKind K, Type *Tp);<br>
+<br>
+ /// Returns the opcode of binary operation corresponding to the ReductionKind.<br>
+ static unsigned getReductionBinOp(ReductionKind Kind);<br>
+<br>
+ /// Returns a Min/Max operation corresponding to MinMaxReductionKind.<br>
+ static Value *createMinMaxOp(IRBuilder<> &Builder,<br>
+ ReductionInstDesc::MinMaxReductionKind RK,<br>
+ Value *Left, Value *Right);<br>
+<br>
+ /// Returns true if Phi is a reduction of type Kind and adds it to the<br>
+ /// ReductionDescriptor.<br>
+ static bool AddReductionVar(PHINode *Phi, ReductionKind Kind, Loop *TheLoop,<br>
+ bool HasFunNoNaNAttr,<br>
+ ReductionDescriptor &RedDes);<br>
+<br>
+ /// Returns true if Phi is a reduction in TheLoop. The ReductionDescriptor is<br>
+ /// returned in RedDes.<br>
+ static bool isReductionPHI(PHINode *Phi, Loop *TheLoop,<br>
+ ReductionDescriptor &RedDes);<br>
+<br>
+ ReductionKind getReductionKind() { return Kind; }<br>
+<br>
+ ReductionInstDesc::MinMaxReductionKind getMinMaxReductionKind() {<br>
+ return MinMaxKind;<br>
+ }<br>
+<br>
+ TrackingVH<Value> getReductionStartValue() { return StartValue; }<br>
+<br>
+ Instruction *getLoopExitInstr() { return LoopExitInstr; }<br>
+<br>
+private:<br>
+ // The starting value of the reduction.<br>
+ // It does not have to be zero!<br>
+ TrackingVH<Value> StartValue;<br>
+ // The instruction who's value is used outside the loop.<br>
+ Instruction *LoopExitInstr;<br>
+ // The kind of the reduction.<br>
+ ReductionKind Kind;<br>
+ // If this a min/max reduction the kind of reduction.<br>
+ ReductionInstDesc::MinMaxReductionKind MinMaxKind;<br>
+};<br>
+<br>
BasicBlock *InsertPreheaderForLoop(Loop *L, Pass *P);<br>
<br>
/// \brief Simplify each loop in a loop nest recursively.<br>
<br>
Modified: llvm/trunk/lib/Transforms/Utils/CMakeLists.txt<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/CMakeLists.txt?rev=235284&r1=235283&r2=235284&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/CMakeLists.txt?rev=235284&r1=235283&r2=235284&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Transforms/Utils/CMakeLists.txt (original)<br>
+++ llvm/trunk/lib/Transforms/Utils/CMakeLists.txt Sun Apr 19 23:38:33 2015<br>
@@ -21,6 +21,7 @@ add_llvm_library(LLVMTransformUtils<br>
LoopSimplify.cpp<br>
LoopUnroll.cpp<br>
LoopUnrollRuntime.cpp<br>
+ LoopUtils.cpp<br>
LowerInvoke.cpp<br>
LowerSwitch.cpp<br>
Mem2Reg.cpp<br>
<br>
Added: llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp?rev=235284&view=auto" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp?rev=235284&view=auto</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp (added)<br>
+++ llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp Sun Apr 19 23:38:33 2015<br>
@@ -0,0 +1,453 @@<br>
+//===-- LoopUtils.cpp - Loop Utility functions -------------------------===//<br>
+//<br>
+// The LLVM Compiler Infrastructure<br>
+//<br>
+// This file is distributed under the University of Illinois Open Source<br>
+// License. See LICENSE.TXT for details.<br>
+//<br>
+//===----------------------------------------------------------------------===//<br>
+//<br>
+// This file defines common loop utility functions.<br>
+//<br>
+//===----------------------------------------------------------------------===//<br>
+<br>
+#include "llvm/Analysis/LoopInfo.h"<br>
+#include "llvm/IR/Instructions.h"<br>
+#include "llvm/IR/PatternMatch.h"<br>
+#include "llvm/IR/ValueHandle.h"<br>
+#include "llvm/Support/Debug.h"<br>
+#include "llvm/Transforms/Utils/LoopUtils.h"<br>
+<br>
+using namespace llvm;<br>
+using namespace llvm::PatternMatch;<br>
+<br>
+#define DEBUG_TYPE "loop-utils"<br>
+<br>
+bool ReductionDescriptor::areAllUsesIn(Instruction *I,<br>
+ SmallPtrSetImpl<Instruction *> &Set) {<br>
+ for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use)<br>
+ if (!Set.count(dyn_cast<Instruction>(*Use)))<br>
+ return false;<br>
+ return true;<br>
+}<br>
+<br>
+bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,<br>
+ Loop *TheLoop, bool HasFunNoNaNAttr,<br>
+ ReductionDescriptor &RedDes) {<br>
+ if (Phi->getNumIncomingValues() != 2)<br>
+ return false;<br>
+<br>
+ // Reduction variables are only found in the loop header block.<br>
+ if (Phi->getParent() != TheLoop->getHeader())<br>
+ return false;<br>
+<br>
+ // Obtain the reduction start value from the value that comes from the loop<br>
+ // preheader.<br>
+ Value *RdxStart = Phi->getIncomingValueForBlock(TheLoop->getLoopPreheader());<br>
+<br>
+ // ExitInstruction is the single value which is used outside the loop.<br>
+ // We only allow for a single reduction value to be used outside the loop.<br>
+ // This includes users of the reduction, variables (which form a cycle<br>
+ // which ends in the phi node).<br>
+ Instruction *ExitInstruction = nullptr;<br>
+ // Indicates that we found a reduction operation in our scan.<br>
+ bool FoundReduxOp = false;<br>
+<br>
+ // We start with the PHI node and scan for all of the users of this<br>
+ // instruction. All users must be instructions that can be used as reduction<br>
+ // variables (such as ADD). We must have a single out-of-block user. The cycle<br>
+ // must include the original PHI.<br>
+ bool FoundStartPHI = false;<br>
+<br>
+ // To recognize min/max patterns formed by a icmp select sequence, we store<br>
+ // the number of instruction we saw from the recognized min/max pattern,<br>
+ // to make sure we only see exactly the two instructions.<br>
+ unsigned NumCmpSelectPatternInst = 0;<br>
+ ReductionInstDesc ReduxDesc(false, nullptr);<br>
+<br>
+ SmallPtrSet<Instruction *, 8> VisitedInsts;<br>
+ SmallVector<Instruction *, 8> Worklist;<br>
+ Worklist.push_back(Phi);<br>
+ VisitedInsts.insert(Phi);<br>
+<br>
+ // A value in the reduction can be used:<br>
+ // - By the reduction:<br>
+ // - Reduction operation:<br>
+ // - One use of reduction value (safe).<br>
+ // - Multiple use of reduction value (not safe).<br>
+ // - PHI:<br>
+ // - All uses of the PHI must be the reduction (safe).<br>
+ // - Otherwise, not safe.<br>
+ // - By one instruction outside of the loop (safe).<br>
+ // - By further instructions outside of the loop (not safe).<br>
+ // - By an instruction that is not part of the reduction (not safe).<br>
+ // This is either:<br>
+ // * An instruction type other than PHI or the reduction operation.<br>
+ // * A PHI in the header other than the initial PHI.<br>
+ while (!Worklist.empty()) {<br>
+ Instruction *Cur = Worklist.back();<br>
+ Worklist.pop_back();<br>
+<br>
+ // No Users.<br>
+ // If the instruction has no users then this is a broken chain and can't be<br>
+ // a reduction variable.<br>
+ if (Cur->use_empty())<br>
+ return false;<br>
+<br>
+ bool IsAPhi = isa<PHINode>(Cur);<br>
+<br>
+ // A header PHI use other than the original PHI.<br>
+ if (Cur != Phi && IsAPhi && Cur->getParent() == Phi->getParent())<br>
+ return false;<br>
+<br>
+ // Reductions of instructions such as Div, and Sub is only possible if the<br>
+ // LHS is the reduction variable.<br>
+ if (!Cur->isCommutative() && !IsAPhi && !isa<SelectInst>(Cur) &&<br>
+ !isa<ICmpInst>(Cur) && !isa<FCmpInst>(Cur) &&<br>
+ !VisitedInsts.count(dyn_cast<Instruction>(Cur->getOperand(0))))<br>
+ return false;<br>
+<br>
+ // Any reduction instruction must be of one of the allowed kinds.<br>
+ ReduxDesc = isReductionInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr);<br>
+ if (!ReduxDesc.isReduction())<br>
+ return false;<br>
+<br>
+ // A reduction operation must only have one use of the reduction value.<br>
+ if (!IsAPhi && Kind != RK_IntegerMinMax && Kind != RK_FloatMinMax &&<br>
+ hasMultipleUsesOf(Cur, VisitedInsts))<br>
+ return false;<br>
+<br>
+ // All inputs to a PHI node must be a reduction value.<br>
+ if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))<br>
+ return false;<br>
+<br>
+ if (Kind == RK_IntegerMinMax &&<br>
+ (isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))<br>
+ ++NumCmpSelectPatternInst;<br>
+ if (Kind == RK_FloatMinMax && (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))<br>
+ ++NumCmpSelectPatternInst;<br>
+<br>
+ // Check whether we found a reduction operator.<br>
+ FoundReduxOp |= !IsAPhi;<br>
+<br>
+ // Process users of current instruction. Push non-PHI nodes after PHI nodes<br>
+ // onto the stack. This way we are going to have seen all inputs to PHI<br>
+ // nodes once we get to them.<br>
+ SmallVector<Instruction *, 8> NonPHIs;<br>
+ SmallVector<Instruction *, 8> PHIs;<br>
+ for (User *U : Cur->users()) {<br>
+ Instruction *UI = cast<Instruction>(U);<br>
+<br>
+ // Check if we found the exit user.<br>
+ BasicBlock *Parent = UI->getParent();<br>
+ if (!TheLoop->contains(Parent)) {<br>
+ // Exit if you find multiple outside users or if the header phi node is<br>
+ // being used. In this case the user uses the value of the previous<br>
+ // iteration, in which case we would loose "VF-1" iterations of the<br>
+ // reduction operation if we vectorize.<br>
+ if (ExitInstruction != nullptr || Cur == Phi)<br>
+ return false;<br>
+<br>
+ // The instruction used by an outside user must be the last instruction<br>
+ // before we feed back to the reduction phi. Otherwise, we loose VF-1<br>
+ // operations on the value.<br>
+ if (std::find(Phi->op_begin(), Phi->op_end(), Cur) == Phi->op_end())<br>
+ return false;<br>
+<br>
+ ExitInstruction = Cur;<br>
+ continue;<br>
+ }<br>
+<br>
+ // Process instructions only once (termination). Each reduction cycle<br>
+ // value must only be used once, except by phi nodes and min/max<br>
+ // reductions which are represented as a cmp followed by a select.<br>
+ ReductionInstDesc IgnoredVal(false, nullptr);<br>
+ if (VisitedInsts.insert(UI).second) {<br>
+ if (isa<PHINode>(UI))<br>
+ PHIs.push_back(UI);<br>
+ else<br>
+ NonPHIs.push_back(UI);<br>
+ } else if (!isa<PHINode>(UI) &&<br>
+ ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) &&<br>
+ !isa<SelectInst>(UI)) ||<br>
+ !isMinMaxSelectCmpPattern(UI, IgnoredVal).isReduction()))<br>
+ return false;<br>
+<br>
+ // Remember that we completed the cycle.<br>
+ if (UI == Phi)<br>
+ FoundStartPHI = true;<br>
+ }<br>
+ Worklist.append(PHIs.begin(), PHIs.end());<br>
+ Worklist.append(NonPHIs.begin(), NonPHIs.end());<br>
+ }<br>
+<br>
+ // This means we have seen one but not the other instruction of the<br>
+ // pattern or more than just a select and cmp.<br>
+ if ((Kind == RK_IntegerMinMax || Kind == RK_FloatMinMax) &&<br>
+ NumCmpSelectPatternInst != 2)<br>
+ return false;<br>
+<br>
+ if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction)<br>
+ return false;<br>
+<br>
+ // We found a reduction var if we have reached the original phi node and we<br>
+ // only have a single instruction with out-of-loop users.<br>
+<br>
+ // The ExitInstruction(Instruction which is allowed to have out-of-loop users)<br>
+ // is saved as part of the ReductionDescriptor.<br>
+<br>
+ // Save the description of this reduction variable.<br>
+ ReductionDescriptor RD(RdxStart, ExitInstruction, Kind,<br>
+ ReduxDesc.getMinMaxKind());<br>
+<br>
+ RedDes = RD;<br>
+<br>
+ return true;<br>
+}<br>
+<br>
+/// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction<br>
+/// pattern corresponding to a min(X, Y) or max(X, Y).<br>
+ReductionInstDesc<br>
+ReductionDescriptor::isMinMaxSelectCmpPattern(Instruction *I,<br>
+ ReductionInstDesc &Prev) {<br>
+<br>
+ assert((isa<ICmpInst>(I) || isa<FCmpInst>(I) || isa<SelectInst>(I)) &&<br>
+ "Expect a select instruction");<br>
+ Instruction *Cmp = nullptr;<br>
+ SelectInst *Select = nullptr;<br>
+<br>
+ // We must handle the select(cmp()) as a single instruction. Advance to the<br>
+ // select.<br>
+ if ((Cmp = dyn_cast<ICmpInst>(I)) || (Cmp = dyn_cast<FCmpInst>(I))) {<br>
+ if (!Cmp->hasOneUse() || !(Select = dyn_cast<SelectInst>(*I->user_begin())))<br>
+ return ReductionInstDesc(false, I);<br>
+ return ReductionInstDesc(Select, Prev.getMinMaxKind());<br>
+ }<br>
+<br>
+ // Only handle single use cases for now.<br>
+ if (!(Select = dyn_cast<SelectInst>(I)))<br>
+ return ReductionInstDesc(false, I);<br>
+ if (!(Cmp = dyn_cast<ICmpInst>(I->getOperand(0))) &&<br>
+ !(Cmp = dyn_cast<FCmpInst>(I->getOperand(0))))<br>
+ return ReductionInstDesc(false, I);<br>
+ if (!Cmp->hasOneUse())<br>
+ return ReductionInstDesc(false, I);<br>
+<br>
+ Value *CmpLeft;<br>
+ Value *CmpRight;<br>
+<br>
+ // Look for a min/max pattern.<br>
+ if (m_UMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMin);<br>
+ else if (m_UMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMax);<br>
+ else if (m_SMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMax);<br>
+ else if (m_SMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMin);<br>
+ else if (m_OrdFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin);<br>
+ else if (m_OrdFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax);<br>
+ else if (m_UnordFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin);<br>
+ else if (m_UnordFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
+ return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax);<br>
+<br>
+ return ReductionInstDesc(false, I);<br>
+}<br>
+<br>
+ReductionInstDesc ReductionDescriptor::isReductionInstr(Instruction *I,<br>
+ ReductionKind Kind,<br>
+ ReductionInstDesc &Prev,<br>
+ bool HasFunNoNaNAttr) {<br>
+ bool FP = I->getType()->isFloatingPointTy();<br>
+ bool FastMath = FP && I->hasUnsafeAlgebra();<br>
+ switch (I->getOpcode()) {<br>
+ default:<br>
+ return ReductionInstDesc(false, I);<br>
+ case Instruction::PHI:<br>
+ if (FP &&<br>
+ (Kind != RK_FloatMult && Kind != RK_FloatAdd && Kind != RK_FloatMinMax))<br>
+ return ReductionInstDesc(false, I);<br>
+ return ReductionInstDesc(I, Prev.getMinMaxKind());<br>
+ case Instruction::Sub:<br>
+ case Instruction::Add:<br>
+ return ReductionInstDesc(Kind == RK_IntegerAdd, I);<br>
+ case Instruction::Mul:<br>
+ return ReductionInstDesc(Kind == RK_IntegerMult, I);<br>
+ case Instruction::And:<br>
+ return ReductionInstDesc(Kind == RK_IntegerAnd, I);<br>
+ case Instruction::Or:<br>
+ return ReductionInstDesc(Kind == RK_IntegerOr, I);<br>
+ case Instruction::Xor:<br>
+ return ReductionInstDesc(Kind == RK_IntegerXor, I);<br>
+ case Instruction::FMul:<br>
+ return ReductionInstDesc(Kind == RK_FloatMult && FastMath, I);<br>
+ case Instruction::FSub:<br>
+ case Instruction::FAdd:<br>
+ return ReductionInstDesc(Kind == RK_FloatAdd && FastMath, I);<br>
+ case Instruction::FCmp:<br>
+ case Instruction::ICmp:<br>
+ case Instruction::Select:<br>
+ if (Kind != RK_IntegerMinMax &&<br>
+ (!HasFunNoNaNAttr || Kind != RK_FloatMinMax))<br>
+ return ReductionInstDesc(false, I);<br>
+ return isMinMaxSelectCmpPattern(I, Prev);<br>
+ }<br>
+}<br>
+<br>
+bool ReductionDescriptor::hasMultipleUsesOf(<br>
+ Instruction *I, SmallPtrSetImpl<Instruction *> &Insts) {<br>
+ unsigned NumUses = 0;<br>
+ for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E;<br>
+ ++Use) {<br>
+ if (Insts.count(dyn_cast<Instruction>(*Use)))<br>
+ ++NumUses;<br>
+ if (NumUses > 1)<br>
+ return true;<br>
+ }<br>
+<br>
+ return false;<br>
+}<br>
+bool ReductionDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,<br>
+ ReductionDescriptor &RedDes) {<br>
+<br>
+ bool HasFunNoNaNAttr = false;<br>
+ BasicBlock *Header = TheLoop->getHeader();<br>
+ Function &F = *Header->getParent();<br>
+ if (F.hasFnAttribute("no-nans-fp-math"))<br>
+ HasFunNoNaNAttr =<br>
+ F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true";<br>
+<br>
+ if (AddReductionVar(Phi, RK_IntegerAdd, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_IntegerMult, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_IntegerOr, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found an OR reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_IntegerAnd, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found an AND reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_IntegerXor, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_IntegerMinMax, TheLoop, HasFunNoNaNAttr,<br>
+ RedDes)) {<br>
+ DEBUG(dbgs() << "Found a MINMAX reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_FloatMult, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_FloatAdd, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ if (AddReductionVar(Phi, RK_FloatMinMax, TheLoop, HasFunNoNaNAttr, RedDes)) {<br>
+ DEBUG(dbgs() << "Found an float MINMAX reduction PHI." << *Phi << "\n");<br>
+ return true;<br>
+ }<br>
+ // Not a reduction of known type.<br>
+ return false;<br>
+}<br>
+<br>
+/// This function returns the identity element (or neutral element) for<br>
+/// the operation K.<br>
+Constant *ReductionDescriptor::getReductionIdentity(ReductionKind K, Type *Tp) {<br>
+ switch (K) {<br>
+ case RK_IntegerXor:<br>
+ case RK_IntegerAdd:<br>
+ case RK_IntegerOr:<br>
+ // Adding, Xoring, Oring zero to a number does not change it.<br>
+ return ConstantInt::get(Tp, 0);<br>
+ case RK_IntegerMult:<br>
+ // Multiplying a number by 1 does not change it.<br>
+ return ConstantInt::get(Tp, 1);<br>
+ case RK_IntegerAnd:<br>
+ // AND-ing a number with an all-1 value does not change it.<br>
+ return ConstantInt::get(Tp, -1, true);<br>
+ case RK_FloatMult:<br>
+ // Multiplying a number by 1 does not change it.<br>
+ return ConstantFP::get(Tp, 1.0L);<br>
+ case RK_FloatAdd:<br>
+ // Adding zero to a number does not change it.<br>
+ return ConstantFP::get(Tp, 0.0L);<br>
+ default:<br>
+ llvm_unreachable("Unknown reduction kind");<br>
+ }<br>
+}<br>
+<br>
+/// This function translates the reduction kind to an LLVM binary operator.<br>
+unsigned ReductionDescriptor::getReductionBinOp(ReductionKind Kind) {<br>
+ switch (Kind) {<br>
+ case RK_IntegerAdd:<br>
+ return Instruction::Add;<br>
+ case RK_IntegerMult:<br>
+ return Instruction::Mul;<br>
+ case RK_IntegerOr:<br>
+ return Instruction::Or;<br>
+ case RK_IntegerAnd:<br>
+ return Instruction::And;<br>
+ case RK_IntegerXor:<br>
+ return Instruction::Xor;<br>
+ case RK_FloatMult:<br>
+ return Instruction::FMul;<br>
+ case RK_FloatAdd:<br>
+ return Instruction::FAdd;<br>
+ case RK_IntegerMinMax:<br>
+ return Instruction::ICmp;<br>
+ case RK_FloatMinMax:<br>
+ return Instruction::FCmp;<br>
+ default:<br>
+ llvm_unreachable("Unknown reduction operation");<br>
+ }<br>
+}<br>
+<br>
+Value *<br>
+ReductionDescriptor::createMinMaxOp(IRBuilder<> &Builder,<br>
+ ReductionInstDesc::MinMaxReductionKind RK,<br>
+ Value *Left, Value *Right) {<br>
+ CmpInst::Predicate P = CmpInst::ICMP_NE;<br>
+ switch (RK) {<br>
+ default:<br>
+ llvm_unreachable("Unknown min/max reduction kind");<br>
+ case ReductionInstDesc::MRK_UIntMin:<br>
+ P = CmpInst::ICMP_ULT;<br>
+ break;<br>
+ case ReductionInstDesc::MRK_UIntMax:<br>
+ P = CmpInst::ICMP_UGT;<br>
+ break;<br>
+ case ReductionInstDesc::MRK_SIntMin:<br>
+ P = CmpInst::ICMP_SLT;<br>
+ break;<br>
+ case ReductionInstDesc::MRK_SIntMax:<br>
+ P = CmpInst::ICMP_SGT;<br>
+ break;<br>
+ case ReductionInstDesc::MRK_FloatMin:<br>
+ P = CmpInst::FCMP_OLT;<br>
+ break;<br>
+ case ReductionInstDesc::MRK_FloatMax:<br>
+ P = CmpInst::FCMP_OGT;<br>
+ break;<br>
+ }<br>
+<br>
+ Value *Cmp;<br>
+ if (RK == ReductionInstDesc::MRK_FloatMin ||<br>
+ RK == ReductionInstDesc::MRK_FloatMax)<br>
+ Cmp = Builder.CreateFCmp(P, Left, Right, "rdx.minmax.cmp");<br>
+ else<br>
+ Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp");<br>
+<br>
+ Value *Select = Builder.CreateSelect(Cmp, Left, Right, "rdx.minmax.select");<br>
+ return Select;<br>
+}<br>
<br>
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=235284&r1=235283&r2=235284&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=235284&r1=235283&r2=235284&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)<br>
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Sun Apr 19 23:38:33 2015<br>
@@ -568,20 +568,6 @@ public:<br>
TTI(TTI), DT(DT), LAA(LAA), LAI(nullptr), Induction(nullptr),<br>
WidestIndTy(nullptr), HasFunNoNaNAttr(false) {}<br>
<br>
- /// This enum represents the kinds of reductions that we support.<br>
- enum ReductionKind {<br>
- RK_NoReduction, ///< Not a reduction.<br>
- RK_IntegerAdd, ///< Sum of integers.<br>
- RK_IntegerMult, ///< Product of integers.<br>
- RK_IntegerOr, ///< Bitwise or logical OR of numbers.<br>
- RK_IntegerAnd, ///< Bitwise or logical AND of numbers.<br>
- RK_IntegerXor, ///< Bitwise or logical XOR of numbers.<br>
- RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()).<br>
- RK_FloatAdd, ///< Sum of floats.<br>
- RK_FloatMult, ///< Product of floats.<br>
- RK_FloatMinMax ///< Min/max implemented in terms of select(cmp()).<br>
- };<br>
-<br>
/// This enum represents the kinds of inductions that we support.<br>
enum InductionKind {<br>
IK_NoInduction, ///< Not an induction variable.<br>
@@ -589,54 +575,6 @@ public:<br>
IK_PtrInduction ///< Pointer induction var. Step = C / sizeof(elem).<br>
};<br>
<br>
- // This enum represents the kind of minmax reduction.<br>
- enum MinMaxReductionKind {<br>
- MRK_Invalid,<br>
- MRK_UIntMin,<br>
- MRK_UIntMax,<br>
- MRK_SIntMin,<br>
- MRK_SIntMax,<br>
- MRK_FloatMin,<br>
- MRK_FloatMax<br>
- };<br>
-<br>
- /// This struct holds information about reduction variables.<br>
- struct ReductionDescriptor {<br>
- ReductionDescriptor() : StartValue(nullptr), LoopExitInstr(nullptr),<br>
- Kind(RK_NoReduction), MinMaxKind(MRK_Invalid) {}<br>
-<br>
- ReductionDescriptor(Value *Start, Instruction *Exit, ReductionKind K,<br>
- MinMaxReductionKind MK)<br>
- : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK) {}<br>
-<br>
- // The starting value of the reduction.<br>
- // It does not have to be zero!<br>
- TrackingVH<Value> StartValue;<br>
- // The instruction who's value is used outside the loop.<br>
- Instruction *LoopExitInstr;<br>
- // The kind of the reduction.<br>
- ReductionKind Kind;<br>
- // If this a min/max reduction the kind of reduction.<br>
- MinMaxReductionKind MinMaxKind;<br>
- };<br>
-<br>
- /// This POD struct holds information about a potential reduction operation.<br>
- struct ReductionInstDesc {<br>
- ReductionInstDesc(bool IsRedux, Instruction *I) :<br>
- IsReduction(IsRedux), PatternLastInst(I), MinMaxKind(MRK_Invalid) {}<br>
-<br>
- ReductionInstDesc(Instruction *I, MinMaxReductionKind K) :<br>
- IsReduction(true), PatternLastInst(I), MinMaxKind(K) {}<br>
-<br>
- // Is this instruction a reduction candidate.<br>
- bool IsReduction;<br>
- // The last instruction in a min/max pattern (select of the select(icmp())<br>
- // pattern), or the current reduction instruction otherwise.<br>
- Instruction *PatternLastInst;<br>
- // If this is a min/max pattern the comparison predicate.<br>
- MinMaxReductionKind MinMaxKind;<br>
- };<br>
-<br>
/// A struct for saving information about induction variables.<br>
struct InductionInfo {<br>
InductionInfo(Value *Start, InductionKind K, ConstantInt *Step)<br>
@@ -759,10 +697,6 @@ public:<br>
return LAI;<br>
}<br>
<br>
- /// This function returns the identity element (or neutral element) for<br>
- /// the operation K.<br>
- static Constant *getReductionIdentity(ReductionKind K, Type *Tp);<br>
-<br>
unsigned getMaxSafeDepDistBytes() { return LAI->getMaxSafeDepDistBytes(); }<br>
<br>
bool hasStride(Value *V) { return StrideSet.count(V); }<br>
@@ -820,20 +754,6 @@ private:<br>
/// and we know that we can read from them without segfault.<br>
bool blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs);<br>
<br>
- /// Returns True, if 'Phi' is the kind of reduction variable for type<br>
- /// 'Kind'. If this is a reduction variable, it adds it to ReductionList.<br>
- bool AddReductionVar(PHINode *Phi, ReductionKind Kind);<br>
- /// Returns a struct describing if the instruction 'I' can be a reduction<br>
- /// variable of type 'Kind'. If the reduction is a min/max pattern of<br>
- /// select(icmp()) this function advances the instruction pointer 'I' from the<br>
- /// compare instruction to the select instruction and stores this pointer in<br>
- /// 'PatternLastInst' member of the returned struct.<br>
- ReductionInstDesc isReductionInstr(Instruction *I, ReductionKind Kind,<br>
- ReductionInstDesc &Desc);<br>
- /// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction<br>
- /// pattern corresponding to a min(X, Y) or max(X, Y).<br>
- static ReductionInstDesc isMinMaxSelectCmpPattern(Instruction *I,<br>
- ReductionInstDesc &Prev);<br>
/// Returns the induction kind of Phi and record the step. This function may<br>
/// return NoInduction if the PHI is not an induction variable.<br>
InductionKind isInductionVariable(PHINode *Phi, ConstantInt *&StepValue);<br>
@@ -2469,98 +2389,6 @@ void InnerLoopVectorizer::createEmptyLoo<br>
Hints.setAlreadyVectorized();<br>
}<br>
<br>
-/// This function returns the identity element (or neutral element) for<br>
-/// the operation K.<br>
-Constant*<br>
-LoopVectorizationLegality::getReductionIdentity(ReductionKind K, Type *Tp) {<br>
- switch (K) {<br>
- case RK_IntegerXor:<br>
- case RK_IntegerAdd:<br>
- case RK_IntegerOr:<br>
- // Adding, Xoring, Oring zero to a number does not change it.<br>
- return ConstantInt::get(Tp, 0);<br>
- case RK_IntegerMult:<br>
- // Multiplying a number by 1 does not change it.<br>
- return ConstantInt::get(Tp, 1);<br>
- case RK_IntegerAnd:<br>
- // AND-ing a number with an all-1 value does not change it.<br>
- return ConstantInt::get(Tp, -1, true);<br>
- case RK_FloatMult:<br>
- // Multiplying a number by 1 does not change it.<br>
- return ConstantFP::get(Tp, 1.0L);<br>
- case RK_FloatAdd:<br>
- // Adding zero to a number does not change it.<br>
- return ConstantFP::get(Tp, 0.0L);<br>
- default:<br>
- llvm_unreachable("Unknown reduction kind");<br>
- }<br>
-}<br>
-<br>
-/// This function translates the reduction kind to an LLVM binary operator.<br>
-static unsigned<br>
-getReductionBinOp(LoopVectorizationLegality::ReductionKind Kind) {<br>
- switch (Kind) {<br>
- case LoopVectorizationLegality::RK_IntegerAdd:<br>
- return Instruction::Add;<br>
- case LoopVectorizationLegality::RK_IntegerMult:<br>
- return Instruction::Mul;<br>
- case LoopVectorizationLegality::RK_IntegerOr:<br>
- return Instruction::Or;<br>
- case LoopVectorizationLegality::RK_IntegerAnd:<br>
- return Instruction::And;<br>
- case LoopVectorizationLegality::RK_IntegerXor:<br>
- return Instruction::Xor;<br>
- case LoopVectorizationLegality::RK_FloatMult:<br>
- return Instruction::FMul;<br>
- case LoopVectorizationLegality::RK_FloatAdd:<br>
- return Instruction::FAdd;<br>
- case LoopVectorizationLegality::RK_IntegerMinMax:<br>
- return Instruction::ICmp;<br>
- case LoopVectorizationLegality::RK_FloatMinMax:<br>
- return Instruction::FCmp;<br>
- default:<br>
- llvm_unreachable("Unknown reduction operation");<br>
- }<br>
-}<br>
-<br>
-static Value *createMinMaxOp(IRBuilder<> &Builder,<br>
- LoopVectorizationLegality::MinMaxReductionKind RK,<br>
- Value *Left, Value *Right) {<br>
- CmpInst::Predicate P = CmpInst::ICMP_NE;<br>
- switch (RK) {<br>
- default:<br>
- llvm_unreachable("Unknown min/max reduction kind");<br>
- case LoopVectorizationLegality::MRK_UIntMin:<br>
- P = CmpInst::ICMP_ULT;<br>
- break;<br>
- case LoopVectorizationLegality::MRK_UIntMax:<br>
- P = CmpInst::ICMP_UGT;<br>
- break;<br>
- case LoopVectorizationLegality::MRK_SIntMin:<br>
- P = CmpInst::ICMP_SLT;<br>
- break;<br>
- case LoopVectorizationLegality::MRK_SIntMax:<br>
- P = CmpInst::ICMP_SGT;<br>
- break;<br>
- case LoopVectorizationLegality::MRK_FloatMin:<br>
- P = CmpInst::FCMP_OLT;<br>
- break;<br>
- case LoopVectorizationLegality::MRK_FloatMax:<br>
- P = CmpInst::FCMP_OGT;<br>
- break;<br>
- }<br>
-<br>
- Value *Cmp;<br>
- if (RK == LoopVectorizationLegality::MRK_FloatMin ||<br>
- RK == LoopVectorizationLegality::MRK_FloatMax)<br>
- Cmp = Builder.CreateFCmp(P, Left, Right, "rdx.minmax.cmp");<br>
- else<br>
- Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp");<br>
-<br>
- Value *Select = Builder.CreateSelect(Cmp, Left, Right, "rdx.minmax.select");<br>
- return Select;<br>
-}<br>
-<br>
namespace {<br>
struct CSEDenseMapInfo {<br>
static bool canHandle(Instruction *I) {<br>
@@ -2772,10 +2600,14 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
// Find the reduction variable descriptor.<br>
assert(Legal->getReductionVars()->count(RdxPhi) &&<br>
"Unable to find the reduction variable");<br>
- LoopVectorizationLegality::ReductionDescriptor RdxDesc =<br>
- (*Legal->getReductionVars())[RdxPhi];<br>
+ ReductionDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi];<br>
<br>
- setDebugLocFromInst(Builder, RdxDesc.StartValue);<br>
+ ReductionDescriptor::ReductionKind RK = RdxDesc.getReductionKind();<br>
+ TrackingVH<Value> ReductionStartValue = RdxDesc.getReductionStartValue();<br>
+ Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();<br>
+ ReductionInstDesc::MinMaxReductionKind MinMaxKind =<br>
+ RdxDesc.getMinMaxReductionKind();<br>
+ setDebugLocFromInst(Builder, ReductionStartValue);<br>
<br>
// We need to generate a reduction vector from the incoming scalar.<br>
// To do so, we need to generate the 'identity' vector and override<br>
@@ -2784,40 +2616,38 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
Builder.SetInsertPoint(LoopBypassBlocks[1]->getTerminator());<br>
<br>
// This is the vector-clone of the value that leaves the loop.<br>
- VectorParts &VectorExit = getVectorValue(RdxDesc.LoopExitInstr);<br>
+ VectorParts &VectorExit = getVectorValue(LoopExitInst);<br>
Type *VecTy = VectorExit[0]->getType();<br>
<br>
// Find the reduction identity variable. Zero for addition, or, xor,<br>
// one for multiplication, -1 for And.<br>
Value *Identity;<br>
Value *VectorStart;<br>
- if (RdxDesc.Kind == LoopVectorizationLegality::RK_IntegerMinMax ||<br>
- RdxDesc.Kind == LoopVectorizationLegality::RK_FloatMinMax) {<br>
+ if (RK == ReductionDescriptor::RK_IntegerMinMax ||<br>
+ RK == ReductionDescriptor::RK_FloatMinMax) {<br>
// MinMax reduction have the start value as their identify.<br>
if (VF == 1) {<br>
- VectorStart = Identity = RdxDesc.StartValue;<br>
+ VectorStart = Identity = ReductionStartValue;<br>
} else {<br>
- VectorStart = Identity = Builder.CreateVectorSplat(VF,<br>
- RdxDesc.StartValue,<br>
- "minmax.ident");<br>
+ VectorStart = Identity =<br>
+ Builder.CreateVectorSplat(VF, ReductionStartValue, "minmax.ident");<br>
}<br>
} else {<br>
// Handle other reduction kinds:<br>
Constant *Iden =<br>
- LoopVectorizationLegality::getReductionIdentity(RdxDesc.Kind,<br>
- VecTy->getScalarType());<br>
+ ReductionDescriptor::getReductionIdentity(RK, VecTy->getScalarType());<br>
if (VF == 1) {<br>
Identity = Iden;<br>
// This vector is the Identity vector where the first element is the<br>
// incoming scalar reduction.<br>
- VectorStart = RdxDesc.StartValue;<br>
+ VectorStart = ReductionStartValue;<br>
} else {<br>
Identity = ConstantVector::getSplat(VF, Iden);<br>
<br>
// This vector is the Identity vector where the first element is the<br>
// incoming scalar reduction.<br>
- VectorStart = Builder.CreateInsertElement(Identity,<br>
- RdxDesc.StartValue, Zero);<br>
+ VectorStart =<br>
+ Builder.CreateInsertElement(Identity, ReductionStartValue, Zero);<br>
}<br>
}<br>
<br>
@@ -2846,11 +2676,11 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
Builder.SetInsertPoint(LoopMiddleBlock->getFirstInsertionPt());<br>
<br>
VectorParts RdxParts;<br>
- setDebugLocFromInst(Builder, RdxDesc.LoopExitInstr);<br>
+ setDebugLocFromInst(Builder, LoopExitInst);<br>
for (unsigned part = 0; part < UF; ++part) {<br>
// This PHINode contains the vectorized reduction variable, or<br>
// the initial value vector, if we bypass the vector loop.<br>
- VectorParts &RdxExitVal = getVectorValue(RdxDesc.LoopExitInstr);<br>
+ VectorParts &RdxExitVal = getVectorValue(LoopExitInst);<br>
PHINode *NewPhi = Builder.CreatePHI(VecTy, 2, "rdx.vec.exit.phi");<br>
Value *StartVal = (part == 0) ? VectorStart : Identity;<br>
for (unsigned I = 1, E = LoopBypassBlocks.size(); I != E; ++I)<br>
@@ -2862,7 +2692,7 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
<br>
// Reduce all of the unrolled parts into a single vector.<br>
Value *ReducedPartRdx = RdxParts[0];<br>
- unsigned Op = getReductionBinOp(RdxDesc.Kind);<br>
+ unsigned Op = ReductionDescriptor::getReductionBinOp(RK);<br>
setDebugLocFromInst(Builder, ReducedPartRdx);<br>
for (unsigned part = 1; part < UF; ++part) {<br>
if (Op != Instruction::ICmp && Op != Instruction::FCmp)<br>
@@ -2871,8 +2701,8 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxParts[part],<br>
ReducedPartRdx, "bin.rdx"));<br>
else<br>
- ReducedPartRdx = createMinMaxOp(Builder, RdxDesc.MinMaxKind,<br>
- ReducedPartRdx, RdxParts[part]);<br>
+ ReducedPartRdx = ReductionDescriptor::createMinMaxOp(<br>
+ Builder, MinMaxKind, ReducedPartRdx, RdxParts[part]);<br>
}<br>
<br>
if (VF > 1) {<br>
@@ -2903,7 +2733,8 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
TmpVec = addFastMathFlag(Builder.CreateBinOp(<br>
(Instruction::BinaryOps)Op, TmpVec, Shuf, "bin.rdx"));<br>
else<br>
- TmpVec = createMinMaxOp(Builder, RdxDesc.MinMaxKind, TmpVec, Shuf);<br>
+ TmpVec = ReductionDescriptor::createMinMaxOp(Builder, MinMaxKind,<br>
+ TmpVec, Shuf);<br>
}<br>
<br>
// The result is in the first element of the vector.<br>
@@ -2915,7 +2746,7 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
// block and the middle block.<br>
PHINode *BCBlockPhi = PHINode::Create(RdxPhi->getType(), 2, "bc.merge.rdx",<br>
LoopScalarPreHeader->getTerminator());<br>
- BCBlockPhi->addIncoming(RdxDesc.StartValue, LoopBypassBlocks[0]);<br>
+ BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[0]);<br>
BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);<br>
<br>
// Now, we need to fix the users of the reduction variable<br>
@@ -2933,7 +2764,7 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
<br>
// We found our reduction value exit-PHI. Update it with the<br>
// incoming bypass edge.<br>
- if (LCSSAPhi->getIncomingValue(0) == RdxDesc.LoopExitInstr) {<br>
+ if (LCSSAPhi->getIncomingValue(0) == LoopExitInst) {<br>
// Add an edge coming from the bypass.<br>
LCSSAPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);<br>
break;<br>
@@ -2948,7 +2779,7 @@ void InnerLoopVectorizer::vectorizeLoop(<br>
// Pick the other block.<br>
int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);<br>
(RdxPhi)->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);<br>
- (RdxPhi)->setIncomingValue(IncomingEdgeBlockIdx, RdxDesc.LoopExitInstr);<br>
+ (RdxPhi)->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);<br>
}// end of for each redux variable.<br>
<br>
fixLCSSAPHIs();<br>
@@ -3712,41 +3543,9 @@ bool LoopVectorizationLegality::canVecto<br>
continue;<br>
}<br>
<br>
- if (AddReductionVar(Phi, RK_IntegerAdd)) {<br>
- DEBUG(dbgs() << "LV: Found an ADD reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_IntegerMult)) {<br>
- DEBUG(dbgs() << "LV: Found a MUL reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_IntegerOr)) {<br>
- DEBUG(dbgs() << "LV: Found an OR reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_IntegerAnd)) {<br>
- DEBUG(dbgs() << "LV: Found an AND reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_IntegerXor)) {<br>
- DEBUG(dbgs() << "LV: Found a XOR reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_IntegerMinMax)) {<br>
- DEBUG(dbgs() << "LV: Found a MINMAX reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_FloatMult)) {<br>
- DEBUG(dbgs() << "LV: Found an FMult reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_FloatAdd)) {<br>
- DEBUG(dbgs() << "LV: Found an FAdd reduction PHI."<< *Phi <<"\n");<br>
- continue;<br>
- }<br>
- if (AddReductionVar(Phi, RK_FloatMinMax)) {<br>
- DEBUG(dbgs() << "LV: Found an float MINMAX reduction PHI."<< *Phi <<<br>
- "\n");<br>
+ if (ReductionDescriptor::isReductionPHI(Phi, TheLoop,<br>
+ Reductions[Phi])) {<br>
+ AllowedExit.insert(Reductions[Phi].getLoopExitInstr());<br>
continue;<br>
}<br>
<br>
@@ -4029,294 +3828,6 @@ bool LoopVectorizationLegality::canVecto<br>
return true;<br>
}<br>
<br>
-static bool hasMultipleUsesOf(Instruction *I,<br>
- SmallPtrSetImpl<Instruction *> &Insts) {<br>
- unsigned NumUses = 0;<br>
- for(User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use) {<br>
- if (Insts.count(dyn_cast<Instruction>(*Use)))<br>
- ++NumUses;<br>
- if (NumUses > 1)<br>
- return true;<br>
- }<br>
-<br>
- return false;<br>
-}<br>
-<br>
-static bool areAllUsesIn(Instruction *I, SmallPtrSetImpl<Instruction *> &Set) {<br>
- for(User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use)<br>
- if (!Set.count(dyn_cast<Instruction>(*Use)))<br>
- return false;<br>
- return true;<br>
-}<br>
-<br>
-bool LoopVectorizationLegality::AddReductionVar(PHINode *Phi,<br>
- ReductionKind Kind) {<br>
- if (Phi->getNumIncomingValues() != 2)<br>
- return false;<br>
-<br>
- // Reduction variables are only found in the loop header block.<br>
- if (Phi->getParent() != TheLoop->getHeader())<br>
- return false;<br>
-<br>
- // Obtain the reduction start value from the value that comes from the loop<br>
- // preheader.<br>
- Value *RdxStart = Phi->getIncomingValueForBlock(TheLoop->getLoopPreheader());<br>
-<br>
- // ExitInstruction is the single value which is used outside the loop.<br>
- // We only allow for a single reduction value to be used outside the loop.<br>
- // This includes users of the reduction, variables (which form a cycle<br>
- // which ends in the phi node).<br>
- Instruction *ExitInstruction = nullptr;<br>
- // Indicates that we found a reduction operation in our scan.<br>
- bool FoundReduxOp = false;<br>
-<br>
- // We start with the PHI node and scan for all of the users of this<br>
- // instruction. All users must be instructions that can be used as reduction<br>
- // variables (such as ADD). We must have a single out-of-block user. The cycle<br>
- // must include the original PHI.<br>
- bool FoundStartPHI = false;<br>
-<br>
- // To recognize min/max patterns formed by a icmp select sequence, we store<br>
- // the number of instruction we saw from the recognized min/max pattern,<br>
- // to make sure we only see exactly the two instructions.<br>
- unsigned NumCmpSelectPatternInst = 0;<br>
- ReductionInstDesc ReduxDesc(false, nullptr);<br>
-<br>
- SmallPtrSet<Instruction *, 8> VisitedInsts;<br>
- SmallVector<Instruction *, 8> Worklist;<br>
- Worklist.push_back(Phi);<br>
- VisitedInsts.insert(Phi);<br>
-<br>
- // A value in the reduction can be used:<br>
- // - By the reduction:<br>
- // - Reduction operation:<br>
- // - One use of reduction value (safe).<br>
- // - Multiple use of reduction value (not safe).<br>
- // - PHI:<br>
- // - All uses of the PHI must be the reduction (safe).<br>
- // - Otherwise, not safe.<br>
- // - By one instruction outside of the loop (safe).<br>
- // - By further instructions outside of the loop (not safe).<br>
- // - By an instruction that is not part of the reduction (not safe).<br>
- // This is either:<br>
- // * An instruction type other than PHI or the reduction operation.<br>
- // * A PHI in the header other than the initial PHI.<br>
- while (!Worklist.empty()) {<br>
- Instruction *Cur = Worklist.back();<br>
- Worklist.pop_back();<br>
-<br>
- // No Users.<br>
- // If the instruction has no users then this is a broken chain and can't be<br>
- // a reduction variable.<br>
- if (Cur->use_empty())<br>
- return false;<br>
-<br>
- bool IsAPhi = isa<PHINode>(Cur);<br>
-<br>
- // A header PHI use other than the original PHI.<br>
- if (Cur != Phi && IsAPhi && Cur->getParent() == Phi->getParent())<br>
- return false;<br>
-<br>
- // Reductions of instructions such as Div, and Sub is only possible if the<br>
- // LHS is the reduction variable.<br>
- if (!Cur->isCommutative() && !IsAPhi && !isa<SelectInst>(Cur) &&<br>
- !isa<ICmpInst>(Cur) && !isa<FCmpInst>(Cur) &&<br>
- !VisitedInsts.count(dyn_cast<Instruction>(Cur->getOperand(0))))<br>
- return false;<br>
-<br>
- // Any reduction instruction must be of one of the allowed kinds.<br>
- ReduxDesc = isReductionInstr(Cur, Kind, ReduxDesc);<br>
- if (!ReduxDesc.IsReduction)<br>
- return false;<br>
-<br>
- // A reduction operation must only have one use of the reduction value.<br>
- if (!IsAPhi && Kind != RK_IntegerMinMax && Kind != RK_FloatMinMax &&<br>
- hasMultipleUsesOf(Cur, VisitedInsts))<br>
- return false;<br>
-<br>
- // All inputs to a PHI node must be a reduction value.<br>
- if(IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))<br>
- return false;<br>
-<br>
- if (Kind == RK_IntegerMinMax && (isa<ICmpInst>(Cur) ||<br>
- isa<SelectInst>(Cur)))<br>
- ++NumCmpSelectPatternInst;<br>
- if (Kind == RK_FloatMinMax && (isa<FCmpInst>(Cur) ||<br>
- isa<SelectInst>(Cur)))<br>
- ++NumCmpSelectPatternInst;<br>
-<br>
- // Check whether we found a reduction operator.<br>
- FoundReduxOp |= !IsAPhi;<br>
-<br>
- // Process users of current instruction. Push non-PHI nodes after PHI nodes<br>
- // onto the stack. This way we are going to have seen all inputs to PHI<br>
- // nodes once we get to them.<br>
- SmallVector<Instruction *, 8> NonPHIs;<br>
- SmallVector<Instruction *, 8> PHIs;<br>
- for (User *U : Cur->users()) {<br>
- Instruction *UI = cast<Instruction>(U);<br>
-<br>
- // Check if we found the exit user.<br>
- BasicBlock *Parent = UI->getParent();<br>
- if (!TheLoop->contains(Parent)) {<br>
- // Exit if you find multiple outside users or if the header phi node is<br>
- // being used. In this case the user uses the value of the previous<br>
- // iteration, in which case we would loose "VF-1" iterations of the<br>
- // reduction operation if we vectorize.<br>
- if (ExitInstruction != nullptr || Cur == Phi)<br>
- return false;<br>
-<br>
- // The instruction used by an outside user must be the last instruction<br>
- // before we feed back to the reduction phi. Otherwise, we loose VF-1<br>
- // operations on the value.<br>
- if (std::find(Phi->op_begin(), Phi->op_end(), Cur) == Phi->op_end())<br>
- return false;<br>
-<br>
- ExitInstruction = Cur;<br>
- continue;<br>
- }<br>
-<br>
- // Process instructions only once (termination). Each reduction cycle<br>
- // value must only be used once, except by phi nodes and min/max<br>
- // reductions which are represented as a cmp followed by a select.<br>
- ReductionInstDesc IgnoredVal(false, nullptr);<br>
- if (VisitedInsts.insert(UI).second) {<br>
- if (isa<PHINode>(UI))<br>
- PHIs.push_back(UI);<br>
- else<br>
- NonPHIs.push_back(UI);<br>
- } else if (!isa<PHINode>(UI) &&<br>
- ((!isa<FCmpInst>(UI) &&<br>
- !isa<ICmpInst>(UI) &&<br>
- !isa<SelectInst>(UI)) ||<br>
- !isMinMaxSelectCmpPattern(UI, IgnoredVal).IsReduction))<br>
- return false;<br>
-<br>
- // Remember that we completed the cycle.<br>
- if (UI == Phi)<br>
- FoundStartPHI = true;<br>
- }<br>
- Worklist.append(PHIs.begin(), PHIs.end());<br>
- Worklist.append(NonPHIs.begin(), NonPHIs.end());<br>
- }<br>
-<br>
- // This means we have seen one but not the other instruction of the<br>
- // pattern or more than just a select and cmp.<br>
- if ((Kind == RK_IntegerMinMax || Kind == RK_FloatMinMax) &&<br>
- NumCmpSelectPatternInst != 2)<br>
- return false;<br>
-<br>
- if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction)<br>
- return false;<br>
-<br>
- // We found a reduction var if we have reached the original phi node and we<br>
- // only have a single instruction with out-of-loop users.<br>
-<br>
- // This instruction is allowed to have out-of-loop users.<br>
- AllowedExit.insert(ExitInstruction);<br>
-<br>
- // Save the description of this reduction variable.<br>
- ReductionDescriptor RD(RdxStart, ExitInstruction, Kind,<br>
- ReduxDesc.MinMaxKind);<br>
- Reductions[Phi] = RD;<br>
- // We've ended the cycle. This is a reduction variable if we have an<br>
- // outside user and it has a binary op.<br>
-<br>
- return true;<br>
-}<br>
-<br>
-/// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction<br>
-/// pattern corresponding to a min(X, Y) or max(X, Y).<br>
-LoopVectorizationLegality::ReductionInstDesc<br>
-LoopVectorizationLegality::isMinMaxSelectCmpPattern(Instruction *I,<br>
- ReductionInstDesc &Prev) {<br>
-<br>
- assert((isa<ICmpInst>(I) || isa<FCmpInst>(I) || isa<SelectInst>(I)) &&<br>
- "Expect a select instruction");<br>
- Instruction *Cmp = nullptr;<br>
- SelectInst *Select = nullptr;<br>
-<br>
- // We must handle the select(cmp()) as a single instruction. Advance to the<br>
- // select.<br>
- if ((Cmp = dyn_cast<ICmpInst>(I)) || (Cmp = dyn_cast<FCmpInst>(I))) {<br>
- if (!Cmp->hasOneUse() || !(Select = dyn_cast<SelectInst>(*I->user_begin())))<br>
- return ReductionInstDesc(false, I);<br>
- return ReductionInstDesc(Select, Prev.MinMaxKind);<br>
- }<br>
-<br>
- // Only handle single use cases for now.<br>
- if (!(Select = dyn_cast<SelectInst>(I)))<br>
- return ReductionInstDesc(false, I);<br>
- if (!(Cmp = dyn_cast<ICmpInst>(I->getOperand(0))) &&<br>
- !(Cmp = dyn_cast<FCmpInst>(I->getOperand(0))))<br>
- return ReductionInstDesc(false, I);<br>
- if (!Cmp->hasOneUse())<br>
- return ReductionInstDesc(false, I);<br>
-<br>
- Value *CmpLeft;<br>
- Value *CmpRight;<br>
-<br>
- // Look for a min/max pattern.<br>
- if (m_UMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_UIntMin);<br>
- else if (m_UMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_UIntMax);<br>
- else if (m_SMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_SIntMax);<br>
- else if (m_SMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_SIntMin);<br>
- else if (m_OrdFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_FloatMin);<br>
- else if (m_OrdFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_FloatMax);<br>
- else if (m_UnordFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_FloatMin);<br>
- else if (m_UnordFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))<br>
- return ReductionInstDesc(Select, MRK_FloatMax);<br>
-<br>
- return ReductionInstDesc(false, I);<br>
-}<br>
-<br>
-LoopVectorizationLegality::ReductionInstDesc<br>
-LoopVectorizationLegality::isReductionInstr(Instruction *I,<br>
- ReductionKind Kind,<br>
- ReductionInstDesc &Prev) {<br>
- bool FP = I->getType()->isFloatingPointTy();<br>
- bool FastMath = FP && I->hasUnsafeAlgebra();<br>
- switch (I->getOpcode()) {<br>
- default:<br>
- return ReductionInstDesc(false, I);<br>
- case Instruction::PHI:<br>
- if (FP && (Kind != RK_FloatMult && Kind != RK_FloatAdd &&<br>
- Kind != RK_FloatMinMax))<br>
- return ReductionInstDesc(false, I);<br>
- return ReductionInstDesc(I, Prev.MinMaxKind);<br>
- case Instruction::Sub:<br>
- case Instruction::Add:<br>
- return ReductionInstDesc(Kind == RK_IntegerAdd, I);<br>
- case Instruction::Mul:<br>
- return ReductionInstDesc(Kind == RK_IntegerMult, I);<br>
- case Instruction::And:<br>
- return ReductionInstDesc(Kind == RK_IntegerAnd, I);<br>
- case Instruction::Or:<br>
- return ReductionInstDesc(Kind == RK_IntegerOr, I);<br>
- case Instruction::Xor:<br>
- return ReductionInstDesc(Kind == RK_IntegerXor, I);<br>
- case Instruction::FMul:<br>
- return ReductionInstDesc(Kind == RK_FloatMult && FastMath, I);<br>
- case Instruction::FSub:<br>
- case Instruction::FAdd:<br>
- return ReductionInstDesc(Kind == RK_FloatAdd && FastMath, I);<br>
- case Instruction::FCmp:<br>
- case Instruction::ICmp:<br>
- case Instruction::Select:<br>
- if (Kind != RK_IntegerMinMax &&<br>
- (!HasFunNoNaNAttr || Kind != RK_FloatMinMax))<br>
- return ReductionInstDesc(false, I);<br>
- return isMinMaxSelectCmpPattern(I, Prev);<br>
- }<br>
-}<br>
-<br>
bool llvm::isInductionPHI(PHINode *Phi, ScalarEvolution *SE,<br>
ConstantInt *&StepValue) {<br>
Type *PhiTy = Phi->getType();<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@cs.uiuc.edu">llvm-commits@cs.uiuc.edu</a><br>
<a href="http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits" target="_blank">http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div>