[llvm] r181144 - LoopVectorize: Add support for floating point min/max reductions
Arnold Schwaighofer
aschwaighofer at apple.com
Mon May 6 07:41:40 PDT 2013
Yes. The patch only handles programmer written min/max functions which are expressed in terms of “select(cmp())”. The commit message might be somewhat misleading.
On May 5, 2013, at 9:30 PM, Owen Anderson <resistor at mac.com> wrote:
> IIRC, a "true" IEEE max/min operation can be considered a reduction even in non-fast-math mode. If either operand is NaN, then it returns the other one. Unfortunately, LLVM doesn't model these operations at the moment.
>
> --Owen
>
> On May 4, 2013, at 6:54 PM, Arnold Schwaighofer <aschwaighofer at apple.com> wrote:
>
>> Author: arnolds
>> Date: Sat May 4 20:54:48 2013
>> New Revision: 181144
>>
>> URL: http://llvm.org/viewvc/llvm-project?rev=181144&view=rev
>> Log:
>> LoopVectorize: Add support for floating point min/max reductions
>>
>> Add support for min/max reductions when "no-nans-float-math" is enabled. This
>> allows us to assume we have ordered floating point math and treat ordered and
>> unordered predicates equally.
>>
>> radar://13723044
>>
>> Modified:
>> llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
>> llvm/trunk/test/Transforms/LoopVectorize/minmax_reduction.ll
>>
>> Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=181144&r1=181143&r2=181144&view=diff
>> ==============================================================================
>> --- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)
>> +++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Sat May 4 20:54:48 2013
>> @@ -335,7 +335,7 @@ public:
>> DominatorTree *DT, TargetTransformInfo* TTI,
>> AliasAnalysis *AA, TargetLibraryInfo *TLI)
>> : TheLoop(L), SE(SE), DL(DL), DT(DT), TTI(TTI), AA(AA), TLI(TLI),
>> - Induction(0) {}
>> + Induction(0), HasFunNoNaNAttr(false) {}
>>
>> /// This enum represents the kinds of reductions that we support.
>> enum ReductionKind {
>> @@ -347,7 +347,8 @@ public:
>> RK_IntegerXor, ///< Bitwise or logical XOR of numbers.
>> RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()).
>> RK_FloatAdd, ///< Sum of floats.
>> - RK_FloatMult ///< Product of floats.
>> + RK_FloatMult, ///< Product of floats.
>> + RK_FloatMinMax ///< Min/max implemented in terms of select(cmp()).
>> };
>>
>> /// This enum represents the kinds of inductions that we support.
>> @@ -365,7 +366,9 @@ public:
>> MRK_UIntMin,
>> MRK_UIntMax,
>> MRK_SIntMin,
>> - MRK_SIntMax
>> + MRK_SIntMax,
>> + MRK_FloatMin,
>> + MRK_FloatMax
>> };
>>
>> /// This POD struct holds information about reduction variables.
>> @@ -586,6 +589,8 @@ private:
>> /// We need to check that all of the pointers in this list are disjoint
>> /// at runtime.
>> RuntimePointerCheck PtrRtCheck;
>> + /// Can we assume the absence of NaNs.
>> + bool HasFunNoNaNAttr;
>> };
>>
>> /// LoopVectorizationCostModel - estimates the expected speedups due to
>> @@ -1648,6 +1653,8 @@ getReductionBinOp(LoopVectorizationLegal
>> return Instruction::FAdd;
>> case LoopVectorizationLegality::RK_IntegerMinMax:
>> return Instruction::ICmp;
>> + case LoopVectorizationLegality::RK_FloatMinMax:
>> + return Instruction::FCmp;
>> default:
>> llvm_unreachable("Unknown reduction operation");
>> }
>> @@ -1672,8 +1679,21 @@ Value *createMinMaxOp(IRBuilder<> &Build
>> break;
>> case LoopVectorizationLegality::MRK_SIntMax:
>> P = CmpInst::ICMP_SGT;
>> + break;
>> + case LoopVectorizationLegality::MRK_FloatMin:
>> + P = CmpInst::FCMP_OLT;
>> + break;
>> + case LoopVectorizationLegality::MRK_FloatMax:
>> + P = CmpInst::FCMP_OGT;
>> + break;
>> }
>> - Value *Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp");
>> +
>> + Value *Cmp;
>> + if (RK == LoopVectorizationLegality::MRK_FloatMin || RK == LoopVectorizationLegality::MRK_FloatMax)
>> + Cmp = Builder.CreateFCmp(P, Left, Right, "rdx.minmax.cmp");
>> + else
>> + Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp");
>> +
>> Value *Select = Builder.CreateSelect(Cmp, Left, Right, "rdx.minmax.select");
>> return Select;
>> }
>> @@ -1743,11 +1763,12 @@ InnerLoopVectorizer::vectorizeLoop(LoopV
>> // one for multiplication, -1 for And.
>> Value *Identity;
>> Value *VectorStart;
>> - if (RdxDesc.Kind == LoopVectorizationLegality::RK_IntegerMinMax)
>> + if (RdxDesc.Kind == LoopVectorizationLegality::RK_IntegerMinMax ||
>> + RdxDesc.Kind == LoopVectorizationLegality::RK_FloatMinMax) {
>> // MinMax reduction have the start value as their identify.
>> VectorStart = Identity = Builder.CreateVectorSplat(VF, RdxDesc.StartValue,
>> "minmax.ident");
>> - else {
>> + } else {
>> Constant *Iden =
>> LoopVectorizationLegality::getReductionIdentity(RdxDesc.Kind,
>> VecTy->getScalarType());
>> @@ -1801,7 +1822,7 @@ InnerLoopVectorizer::vectorizeLoop(LoopV
>> Value *ReducedPartRdx = RdxParts[0];
>> unsigned Op = getReductionBinOp(RdxDesc.Kind);
>> for (unsigned part = 1; part < UF; ++part) {
>> - if (Op != Instruction::ICmp)
>> + if (Op != Instruction::ICmp && Op != Instruction::FCmp)
>> ReducedPartRdx = Builder.CreateBinOp((Instruction::BinaryOps)Op,
>> RdxParts[part], ReducedPartRdx,
>> "bin.rdx");
>> @@ -1832,7 +1853,7 @@ InnerLoopVectorizer::vectorizeLoop(LoopV
>> ConstantVector::get(ShuffleMask),
>> "rdx.shuf");
>>
>> - if (Op != Instruction::ICmp)
>> + if (Op != Instruction::ICmp && Op != Instruction::FCmp)
>> TmpVec = Builder.CreateBinOp((Instruction::BinaryOps)Op, TmpVec, Shuf,
>> "bin.rdx");
>> else
>> @@ -2363,6 +2384,13 @@ bool LoopVectorizationLegality::canVecto
>> return false;
>> }
>>
>> + // Look for the attribute signaling the absence of NaNs.
>> + Function &F = *Header->getParent();
>> + if (F.hasFnAttribute("no-nans-fp-math"))
>> + HasFunNoNaNAttr = F.getAttributes().getAttribute(
>> + AttributeSet::FunctionIndex,
>> + "no-nans-fp-math").getValueAsString() == "true";
>> +
>> // For each block in the loop.
>> for (Loop::block_iterator bb = TheLoop->block_begin(),
>> be = TheLoop->block_end(); bb != be; ++bb) {
>> @@ -2444,6 +2472,10 @@ bool LoopVectorizationLegality::canVecto
>> DEBUG(dbgs() << "LV: Found an FAdd reduction PHI."<< *Phi <<"\n");
>> continue;
>> }
>> + if (AddReductionVar(Phi, RK_FloatMinMax)) {
>> + DEBUG(dbgs() << "LV: Found an float MINMAX reduction PHI."<< *Phi <<"\n");
>> + continue;
>> + }
>>
>> DEBUG(dbgs() << "LV: Found an unidentified PHI."<< *Phi <<"\n");
>> return false;
>> @@ -2869,7 +2901,7 @@ bool LoopVectorizationLegality::AddReduc
>> // such that we don't stop when we see the phi has two uses (one by the select
>> // and one by the icmp) and to make sure we only see exactly the two
>> // instructions.
>> - unsigned NumICmpSelectPatternInst = 0;
>> + unsigned NumCmpSelectPatternInst = 0;
>> ReductionInstDesc ReduxDesc(false, 0);
>>
>> // Avoid cycles in the chain.
>> @@ -2918,7 +2950,7 @@ bool LoopVectorizationLegality::AddReduc
>>
>> // We can't have multiple inside users except for a combination of
>> // icmp/select both using the phi.
>> - if (FoundInBlockUser && !NumICmpSelectPatternInst)
>> + if (FoundInBlockUser && !NumCmpSelectPatternInst)
>> return false;
>> FoundInBlockUser = true;
>>
>> @@ -2927,14 +2959,15 @@ bool LoopVectorizationLegality::AddReduc
>> if (!ReduxDesc.IsReduction)
>> return false;
>>
>> - if (Kind == RK_IntegerMinMax && (isa<ICmpInst>(U) ||
>> - isa<SelectInst>(U)))
>> - ++NumICmpSelectPatternInst;
>> + if (Kind == RK_IntegerMinMax && (isa<ICmpInst>(U) || isa<SelectInst>(U)))
>> + ++NumCmpSelectPatternInst;
>> + if (Kind == RK_FloatMinMax && (isa<FCmpInst>(U) || isa<SelectInst>(U)))
>> + ++NumCmpSelectPatternInst;
>>
>> // Reductions of instructions such as Div, and Sub is only
>> // possible if the LHS is the reduction variable.
>> if (!U->isCommutative() && !isa<PHINode>(U) && !isa<SelectInst>(U) &&
>> - !isa<ICmpInst>(U) && U->getOperand(0) != Iter)
>> + !isa<ICmpInst>(U) && !isa<FCmpInst>(U) && U->getOperand(0) != Iter)
>> return false;
>>
>> Iter = ReduxDesc.PatternLastInst;
>> @@ -2942,7 +2975,8 @@ bool LoopVectorizationLegality::AddReduc
>>
>> // This means we have seen one but not the other instruction of the
>> // pattern or more than just a select and cmp.
>> - if (Kind == RK_IntegerMinMax && NumICmpSelectPatternInst != 2)
>> + if ((Kind == RK_IntegerMinMax || Kind == RK_FloatMinMax) &&
>> + NumCmpSelectPatternInst != 2)
>> return false;
>>
>> // We found a reduction var if we have reached the original
>> @@ -2968,16 +3002,17 @@ bool LoopVectorizationLegality::AddReduc
>> /// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction
>> /// pattern corresponding to a min(X, Y) or max(X, Y).
>> LoopVectorizationLegality::ReductionInstDesc
>> -LoopVectorizationLegality::isMinMaxSelectCmpPattern(Instruction *I, ReductionInstDesc &Prev) {
>> +LoopVectorizationLegality::isMinMaxSelectCmpPattern(Instruction *I,
>> + ReductionInstDesc &Prev) {
>>
>> - assert((isa<ICmpInst>(I) || isa<SelectInst>(I)) &&
>> + assert((isa<ICmpInst>(I) || isa<FCmpInst>(I) || isa<SelectInst>(I)) &&
>> "Expect a select instruction");
>> - ICmpInst *Cmp = 0;
>> + Instruction *Cmp = 0;
>> SelectInst *Select = 0;
>>
>> // We must handle the select(cmp()) as a single instruction. Advance to the
>> // select.
>> - if ((Cmp = dyn_cast<ICmpInst>(I))) {
>> + if ((Cmp = dyn_cast<ICmpInst>(I)) || (Cmp = dyn_cast<FCmpInst>(I))) {
>> if (!Cmp->hasOneUse() || !(Select = dyn_cast<SelectInst>(*I->use_begin())))
>> return ReductionInstDesc(false, I);
>> return ReductionInstDesc(Select, Prev.MinMaxKind);
>> @@ -2986,7 +3021,8 @@ LoopVectorizationLegality::isMinMaxSelec
>> // Only handle single use cases for now.
>> if (!(Select = dyn_cast<SelectInst>(I)))
>> return ReductionInstDesc(false, I);
>> - if (!(Cmp = dyn_cast<ICmpInst>(I->getOperand(0))))
>> + if (!(Cmp = dyn_cast<ICmpInst>(I->getOperand(0))) &&
>> + !(Cmp = dyn_cast<FCmpInst>(I->getOperand(0))))
>> return ReductionInstDesc(false, I);
>> if (!Cmp->hasOneUse())
>> return ReductionInstDesc(false, I);
>> @@ -3003,6 +3039,14 @@ LoopVectorizationLegality::isMinMaxSelec
>> return ReductionInstDesc(Select, MRK_SIntMax);
>> else if (m_SMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
>> return ReductionInstDesc(Select, MRK_SIntMin);
>> + else if (m_OrdFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
>> + return ReductionInstDesc(Select, MRK_FloatMin);
>> + else if (m_OrdFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
>> + return ReductionInstDesc(Select, MRK_FloatMax);
>> + else if (m_UnordFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
>> + return ReductionInstDesc(Select, MRK_FloatMin);
>> + else if (m_UnordFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
>> + return ReductionInstDesc(Select, MRK_FloatMax);
>>
>> return ReductionInstDesc(false, I);
>> }
>> @@ -3017,7 +3061,8 @@ LoopVectorizationLegality::isReductionIn
>> default:
>> return ReductionInstDesc(false, I);
>> case Instruction::PHI:
>> - if (FP && (Kind != RK_FloatMult && Kind != RK_FloatAdd))
>> + if (FP && (Kind != RK_FloatMult && Kind != RK_FloatAdd &&
>> + Kind != RK_FloatMinMax))
>> return ReductionInstDesc(false, I);
>> return ReductionInstDesc(I, Prev.MinMaxKind);
>> case Instruction::Sub:
>> @@ -3035,9 +3080,11 @@ LoopVectorizationLegality::isReductionIn
>> return ReductionInstDesc(Kind == RK_FloatMult && FastMath, I);
>> case Instruction::FAdd:
>> return ReductionInstDesc(Kind == RK_FloatAdd && FastMath, I);
>> + case Instruction::FCmp:
>> case Instruction::ICmp:
>> case Instruction::Select:
>> - if (Kind != RK_IntegerMinMax)
>> + if (Kind != RK_IntegerMinMax &&
>> + (!HasFunNoNaNAttr || Kind != RK_FloatMinMax))
>> return ReductionInstDesc(false, I);
>> return isMinMaxSelectCmpPattern(I, Prev);
>> }
>>
>> Modified: llvm/trunk/test/Transforms/LoopVectorize/minmax_reduction.ll
>> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/minmax_reduction.ll?rev=181144&r1=181143&r2=181144&view=diff
>> ==============================================================================
>> --- llvm/trunk/test/Transforms/LoopVectorize/minmax_reduction.ll (original)
>> +++ llvm/trunk/test/Transforms/LoopVectorize/minmax_reduction.ll Sat May 4 20:54:48 2013
>> @@ -3,6 +3,8 @@
>> target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
>>
>> @A = common global [1024 x i32] zeroinitializer, align 16
>> + at fA = common global [1024 x float] zeroinitializer, align 16
>> + at dA = common global [1024 x double] zeroinitializer, align 16
>>
>> ; Signed tests.
>>
>> @@ -403,3 +405,481 @@ for.body:
>> for.end:
>> ret i32 %max.red.0
>> }
>> +
>> +; Float tests.
>> +
>> +; Maximum.
>> +
>> +; Turn this into a max reduction in the presence of a no-nans-fp-math attribute.
>> +; CHECK: @max_red_float
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @max_red_float(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ogt float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; CHECK: @max_red_float_ge
>> +; CHECK: fcmp oge <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @max_red_float_ge(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp oge float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; CHECK: @inverted_max_red_float
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_max_red_float(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp olt float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; CHECK: @inverted_max_red_float_le
>> +; CHECK: fcmp ole <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_max_red_float_le(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ole float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; CHECK: @unordered_max_red
>> +; CHECK: fcmp ugt <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @unordered_max_red_float(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ugt float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; CHECK: @unordered_max_red_float_ge
>> +; CHECK: fcmp uge <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @unordered_max_red_float_ge(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp uge float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; CHECK: @inverted_unordered_max_red
>> +; CHECK: fcmp ult <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_unordered_max_red_float(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ult float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; CHECK: @inverted_unordered_max_red_float_le
>> +; CHECK: fcmp ule <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_unordered_max_red_float_le(float %max) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ule float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +; Minimum.
>> +
>> +; Turn this into a min reduction in the presence of a no-nans-fp-math attribute.
>> +; CHECK: @min_red_float
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @min_red_float(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp olt float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; CHECK: @min_red_float_le
>> +; CHECK: fcmp ole <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @min_red_float_le(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ole float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; CHECK: @inverted_min_red_float
>> +; CHECK: fcmp ogt <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_min_red_float(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ogt float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; CHECK: @inverted_min_red_float_ge
>> +; CHECK: fcmp oge <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_min_red_float_ge(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp oge float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; CHECK: @unordered_min_red
>> +; CHECK: fcmp ult <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @unordered_min_red_float(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ult float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; CHECK: @unordered_min_red_float_le
>> +; CHECK: fcmp ule <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @unordered_min_red_float_le(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ule float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; CHECK: @inverted_unordered_min_red
>> +; CHECK: fcmp ugt <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_unordered_min_red_float(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ugt float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; CHECK: @inverted_unordered_min_red_float_ge
>> +; CHECK: fcmp uge <2 x float>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x float>
>> +; CHECK: select <2 x i1>
>> +
>> +define float @inverted_unordered_min_red_float_ge(float %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp uge float %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %min.red.0
>> +}
>> +
>> +; Make sure we handle doubles, too.
>> +; CHECK: @min_red_double
>> +; CHECK: fcmp olt <2 x double>
>> +; CHECK: select <2 x i1>
>> +; CHECK: middle.block
>> +; CHECK: fcmp olt <2 x double>
>> +; CHECK: select <2 x i1>
>> +
>> +define double @min_red_double(double %min) #0 {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %min.red.08 = phi double [ %min, %entry ], [ %min.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x double]* @dA, i64 0, i64 %indvars.iv
>> + %0 = load double* %arrayidx, align 4
>> + %cmp3 = fcmp olt double %0, %min.red.08
>> + %min.red.0 = select i1 %cmp3, double %0, double %min.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret double %min.red.0
>> +}
>> +
>> +
>> +; Don't this into a max reduction. The no-nans-fp-math attribute is missing
>> +; CHECK: @max_red_float_nans
>> +; CHECK-NOT: <2 x float>
>> +
>> +define float @max_red_float_nans(float %max) {
>> +entry:
>> + br label %for.body
>> +
>> +for.body:
>> + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
>> + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
>> + %arrayidx = getelementptr inbounds [1024 x float]* @fA, i64 0, i64 %indvars.iv
>> + %0 = load float* %arrayidx, align 4
>> + %cmp3 = fcmp ogt float %0, %max.red.08
>> + %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
>> + %indvars.iv.next = add i64 %indvars.iv, 1
>> + %exitcond = icmp eq i64 %indvars.iv.next, 1024
>> + br i1 %exitcond, label %for.end, label %for.body
>> +
>> +for.end:
>> + ret float %max.red.0
>> +}
>> +
>> +
>> +attributes #0 = { "no-nans-fp-math"="true" }
>>
>>
>> _______________________________________________
>> llvm-commits mailing list
>> llvm-commits at cs.uiuc.edu
>> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list