[llvm-branch-commits] [llvm] d568cff - [LoopVectorizer][SVE] Vectorize a simple loop with with a scalable VF.
Sander de Smalen via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Dec 9 03:30:18 PST 2020
Author: Sander de Smalen
Date: 2020-12-09T11:25:21Z
New Revision: d568cff696e8fb89ce1b040561c037412767af60
URL: https://github.com/llvm/llvm-project/commit/d568cff696e8fb89ce1b040561c037412767af60
DIFF: https://github.com/llvm/llvm-project/commit/d568cff696e8fb89ce1b040561c037412767af60.diff
LOG: [LoopVectorizer][SVE] Vectorize a simple loop with with a scalable VF.
* Steps are scaled by `vscale`, a runtime value.
* Changes to circumvent the cost-model for now (temporary)
so that the cost-model can be implemented separately.
This can vectorize the following loop [1]:
void loop(int N, double *a, double *b) {
#pragma clang loop vectorize_width(4, scalable)
for (int i = 0; i < N; i++) {
a[i] = b[i] + 1.0;
}
}
[1] This source-level example is based on the pragma proposed
separately in D89031. This patch only implements the LLVM part.
Reviewed By: dmgreen
Differential Revision: https://reviews.llvm.org/D91077
Added:
llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
Modified:
llvm/include/llvm/IR/IRBuilder.h
llvm/lib/IR/IRBuilder.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/test/Transforms/LoopVectorize/metadata-width.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index db215094a7e49..c2b3446d159f2 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -879,6 +879,10 @@ class IRBuilderBase {
Type *ResultType,
const Twine &Name = "");
+ /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
+ /// will be the same type as that of \p Scaling.
+ Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
+
/// Create a call to intrinsic \p ID with 1 operand which is mangled on its
/// type.
CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index c0e4451f52003..f936f5756b6f0 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -80,6 +80,17 @@ static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
return CI;
}
+Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
+ Module *M = GetInsertBlock()->getParent()->getParent();
+ assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
+ Function *TheFn =
+ Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
+ CallInst *CI = createCallHelper(TheFn, {}, this, Name);
+ return cast<ConstantInt>(Scaling)->getSExtValue() == 1
+ ? CI
+ : CreateMul(CI, Scaling);
+}
+
CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
MaybeAlign Align, bool isVolatile,
MDNode *TBAATag, MDNode *ScopeTag,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f504afd1ffc41..a91fb988badf6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1121,6 +1121,15 @@ static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
return R;
}
+/// Return a value for Step multiplied by VF.
+static Value *createStepForVF(IRBuilder<> &B, Constant *Step, ElementCount VF) {
+ assert(isa<ConstantInt>(Step) && "Expected an integer step");
+ Constant *StepVal = ConstantInt::get(
+ Step->getType(),
+ cast<ConstantInt>(Step)->getSExtValue() * VF.getKnownMinValue());
+ return VF.isScalable() ? B.CreateVScale(StepVal) : StepVal;
+}
+
namespace llvm {
void reportVectorizationFailure(const StringRef DebugMsg,
@@ -2277,8 +2286,6 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
const InductionDescriptor &ID) {
// We shouldn't have to build scalar steps if we aren't vectorizing.
assert(VF.isVector() && "VF should be greater than one");
- assert(!VF.isScalable() &&
- "the code below assumes a fixed number of elements at compile time");
// Get the value type and ensure it and the step have the same integer type.
Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
assert(ScalarIVTy == Step->getType() &&
@@ -2303,11 +2310,24 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF)
? 1
: VF.getKnownMinValue();
+ assert((!VF.isScalable() || Lanes == 1) &&
+ "Should never scalarize a scalable vector");
// Compute the scalar steps and save the results in VectorLoopValueMap.
for (unsigned Part = 0; Part < UF; ++Part) {
for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
- auto *StartIdx = getSignedIntOrFpConstant(
- ScalarIVTy, VF.getKnownMinValue() * Part + Lane);
+ auto *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
+ ScalarIVTy->getScalarSizeInBits());
+ Value *StartIdx =
+ createStepForVF(Builder, ConstantInt::get(IntStepTy, Part), VF);
+ if (ScalarIVTy->isFloatingPointTy())
+ StartIdx = Builder.CreateSIToFP(StartIdx, ScalarIVTy);
+ StartIdx = addFastMathFlag(Builder.CreateBinOp(
+ AddOp, StartIdx, getSignedIntOrFpConstant(ScalarIVTy, Lane)));
+ // The step returned by `createStepForVF` is a runtime-evaluated value
+ // when VF is scalable. Otherwise, it should be folded into a Constant.
+ assert((VF.isScalable() || isa<Constant>(StartIdx)) &&
+ "Expected StartIdx to be folded to a constant when VF is not "
+ "scalable");
auto *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, StartIdx, Step));
auto *Add = addFastMathFlag(Builder.CreateBinOp(AddOp, ScalarIV, Mul));
VectorLoopValueMap.setScalarValue(EntryVal, {Part, Lane}, Add);
@@ -2350,10 +2370,11 @@ Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) {
// is known to be uniform after vectorization, this corresponds to lane zero
// of the Part unroll iteration. Otherwise, the last instruction is the one
// we created for the last vector lane of the Part unroll iteration.
- assert(!VF.isScalable() && "scalable vectors not yet supported.");
unsigned LastLane = Cost->isUniformAfterVectorization(I, VF)
? 0
: VF.getKnownMinValue() - 1;
+ assert((!VF.isScalable() || LastLane == 0) &&
+ "Scalable vectorization can't lead to any scalarized values.");
auto *LastInst = cast<Instruction>(
VectorLoopValueMap.getScalarValue(V, {Part, LastLane}));
@@ -2695,7 +2716,6 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
Type *ScalarDataTy = getMemInstValueType(Instr);
- assert(!VF.isScalable() && "scalable vectors not yet supported.");
auto *DataTy = VectorType::get(ScalarDataTy, VF);
const Align Alignment = getLoadStoreAlignment(Instr);
@@ -2728,6 +2748,9 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
InBounds = gep->isInBounds();
if (Reverse) {
+ assert(!VF.isScalable() &&
+ "Reversing vectors is not yet supported for scalable vectors.");
+
// If the address is consecutive but reversed, then the
// wide store needs to start at the last vector element.
PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
@@ -2739,8 +2762,9 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]);
} else {
- PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
- ScalarDataTy, Ptr, Builder.getInt32(Part * VF.getKnownMinValue())));
+ Value *Increment = createStepForVF(Builder, Builder.getInt32(Part), VF);
+ PartPtr = cast<GetElementPtrInst>(
+ Builder.CreateGEP(ScalarDataTy, Ptr, Increment));
PartPtr->setIsInBounds(InBounds);
}
@@ -2945,8 +2969,7 @@ Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
Type *Ty = TC->getType();
// This is where we can make the step a runtime constant.
- assert(!VF.isScalable() && "scalable vectorization is not supported yet");
- Constant *Step = ConstantInt::get(Ty, VF.getKnownMinValue() * UF);
+ Value *Step = createStepForVF(Builder, ConstantInt::get(Ty, UF), VF);
// If the tail is to be folded by masking, round the number of iterations N
// up to a multiple of Step instead of rounding down. This is done by first
@@ -2957,6 +2980,8 @@ Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
if (Cost->foldTailByMasking()) {
assert(isPowerOf2_32(VF.getKnownMinValue() * UF) &&
"VF*UF must be a power of 2 when folding tail by masking");
+ assert(!VF.isScalable() &&
+ "Tail folding not yet supported for scalable vectors");
TC = Builder.CreateAdd(
TC, ConstantInt::get(Ty, VF.getKnownMinValue() * UF - 1), "n.rnd.up");
}
@@ -3035,11 +3060,9 @@ void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
// If tail is to be folded, vector loop takes care of all iterations.
Value *CheckMinIters = Builder.getFalse();
if (!Cost->foldTailByMasking()) {
- assert(!VF.isScalable() && "scalable vectors not yet supported.");
- CheckMinIters = Builder.CreateICmp(
- P, Count,
- ConstantInt::get(Count->getType(), VF.getKnownMinValue() * UF),
- "min.iters.check");
+ Value *Step =
+ createStepForVF(Builder, ConstantInt::get(Count->getType(), UF), VF);
+ CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
}
// Create new preheader for vector loop.
LoopVectorPreHeader =
@@ -3518,8 +3541,8 @@ BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
Value *StartIdx = ConstantInt::get(IdxTy, 0);
// The loop step is equal to the vectorization factor (num of SIMD elements)
// times the unroll factor (num of SIMD instructions).
- assert(!VF.isScalable() && "scalable vectors not yet supported.");
- Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
+ Builder.SetInsertPoint(&*Lp->getHeader()->getFirstInsertionPt());
+ Value *Step = createStepForVF(Builder, ConstantInt::get(IdxTy, UF), VF);
Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
Induction =
createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
@@ -4365,7 +4388,6 @@ void InnerLoopVectorizer::clearReductionWrapFlags(
}
void InnerLoopVectorizer::fixLCSSAPHIs() {
- assert(!VF.isScalable() && "the code below assumes fixed width vectors");
for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
if (LCSSAPhi.getNumIncomingValues() == 1) {
auto *IncomingValue = LCSSAPhi.getIncomingValue(0);
@@ -4376,6 +4398,8 @@ void InnerLoopVectorizer::fixLCSSAPHIs() {
cast<Instruction>(IncomingValue), VF)
? 0
: VF.getKnownMinValue() - 1;
+ assert((!VF.isScalable() || LastLane == 0) &&
+ "scalable vectors dont support non-uniform scalars yet");
// Can be a loop invariant incoming value or the last scalar value to be
// extracted from the vectorized loop.
Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
@@ -5528,7 +5552,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
ElementCount
LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
ElementCount UserVF) {
- assert(!UserVF.isScalable() && "scalable vectorization not yet handled");
MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
unsigned SmallestType, WidestType;
std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
@@ -5541,6 +5564,11 @@ LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
unsigned MaxSafeVectorWidthInBits = Legal->getMaxSafeVectorWidthInBits();
if (UserVF.isNonZero()) {
+ // For now, don't verify legality of scalable vectors.
+ // This will be addressed properly in https://reviews.llvm.org/D91718.
+ if (UserVF.isScalable())
+ return UserVF;
+
// If legally unsafe, clamp the user vectorization factor to a safe value.
unsigned MaxSafeVF = PowerOf2Floor(MaxSafeVectorWidthInBits / WidestType);
if (UserVF.getFixedValue() <= MaxSafeVF)
@@ -5629,6 +5657,9 @@ LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
VectorizationFactor
LoopVectorizationCostModel::selectVectorizationFactor(ElementCount MaxVF) {
+ // FIXME: This can be fixed for scalable vectors later, because at this stage
+ // the LoopVectorizer will only consider vectorizing a loop with scalable
+ // vectors when the loop has a hint to enable vectorization for a given VF.
assert(!MaxVF.isScalable() && "scalable vectors not yet supported");
float Cost = expectedCost(ElementCount::getFixed(1)).first;
@@ -5938,7 +5969,6 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
}
// Clamp the interleave ranges to reasonable counts.
- assert(!VF.isScalable() && "scalable vectors not yet supported.");
unsigned MaxInterleaveCount =
TTI.getMaxInterleaveFactor(VF.getKnownMinValue());
@@ -5954,6 +5984,13 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
// If trip count is known or estimated compile time constant, limit the
// interleave count to be less than the trip count divided by VF, provided it
// is at least 1.
+ //
+ // For scalable vectors we can't know if interleaving is beneficial. It may
+ // not be beneficial for small loops if none of the lanes in the second vector
+ // iterations is enabled. However, for larger loops, there is likely to be a
+ // similar benefit as for fixed-width vectors. For now, we choose to leave
+ // the InterleaveCount as if vscale is '1', although if some information about
+ // the vector is known (e.g. min vector size), we can make a better decision.
if (BestKnownTC) {
MaxInterleaveCount =
std::min(*BestKnownTC / VF.getKnownMinValue(), MaxInterleaveCount);
@@ -5997,7 +6034,7 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
// potentially expose ILP opportunities.
LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
<< "LV: IC is " << IC << '\n'
- << "LV: VF is " << VF.getKnownMinValue() << '\n');
+ << "LV: VF is " << VF << '\n');
const bool AggressivelyInterleaveReductions =
TTI.enableAggressiveInterleaving(HasReductions);
if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {
@@ -6664,8 +6701,6 @@ unsigned LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
LoopVectorizationCostModel::VectorizationCostTy
LoopVectorizationCostModel::getInstructionCost(Instruction *I,
ElementCount VF) {
- assert(!VF.isScalable() &&
- "the cost model is not yet implemented for scalable vectorization");
// If we know that this instruction will remain uniform, check the cost of
// the scalar version.
if (isUniformAfterVectorization(I, VF))
@@ -6729,7 +6764,6 @@ unsigned LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
}
void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
- assert(!VF.isScalable() && "scalable vectors not yet supported.");
if (VF.isScalar())
return;
NumPredStores = 0;
@@ -7316,7 +7350,6 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
Optional<VectorizationFactor>
LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
- assert(!UserVF.isScalable() && "scalable vectorization not yet handled");
assert(OrigLoop->isInnermost() && "Inner loop expected.");
Optional<ElementCount> MaybeMaxVF = CM.computeMaxVF(UserVF, UserIC);
if (!MaybeMaxVF) // Cases that should not to be vectorized nor interleaved.
@@ -7339,9 +7372,9 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
ElementCount MaxVF = MaybeMaxVF.getValue();
assert(MaxVF.isNonZero() && "MaxVF is zero.");
- if (!UserVF.isZero() && UserVF.getFixedValue() <= MaxVF.getFixedValue()) {
+ if (!UserVF.isZero() && ElementCount::isKnownLE(UserVF, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
- assert(isPowerOf2_32(UserVF.getFixedValue()) &&
+ assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
"VF needs to be a power of two");
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
@@ -7352,6 +7385,9 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
return {{UserVF, 0}};
}
+ assert(!MaxVF.isScalable() &&
+ "Scalable vectors not yet supported beyond this point");
+
for (ElementCount VF = ElementCount::getFixed(1);
ElementCount::isKnownLE(VF, MaxVF); VF *= 2) {
// Collect Uniform and Scalar instructions after vectorization with VF.
@@ -8695,6 +8731,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
void VPReplicateRecipe::execute(VPTransformState &State) {
if (State.Instance) { // Generate a single instance.
+ assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this,
*State.Instance, IsPredicated, State);
// Insert scalar instance packing it into a vector.
@@ -8717,6 +8754,8 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
// instruction is uniform inwhich case generate only the first lane for each
// of the UF parts.
unsigned EndLane = IsUniform ? 1 : State.VF.getKnownMinValue();
+ assert((!State.VF.isScalable() || IsUniform) &&
+ "Can't scalarize a scalable vector");
for (unsigned Part = 0; Part < State.UF; ++Part)
for (unsigned Lane = 0; Lane < EndLane; ++Lane)
State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this, {Part, Lane},
@@ -8870,12 +8909,6 @@ static bool processLoopInVPlanNativePath(
// Get user vectorization factor.
ElementCount UserVF = Hints.getWidth();
- if (UserVF.isScalable()) {
- // TODO: Use scalable UserVF once we've added initial support for scalable
- // vectorization. For now we convert it to fixed width, but this will be
- // removed in a later patch.
- UserVF = ElementCount::getFixed(UserVF.getKnownMinValue());
- }
// Plan how to best vectorize, return the best VF and its cost.
const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF);
@@ -9041,13 +9074,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
- if (UserVF.isScalable()) {
- // TODO: Use scalable UserVF once we've added initial support for scalable
- // vectorization. For now we convert it to fixed width, but this will be
- // removed in a later patch.
- UserVF = ElementCount::getFixed(UserVF.getKnownMinValue());
- }
-
unsigned UserIC = Hints.getInterleave();
// Plan how to best vectorize, return the best VF and its cost.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1dd81fa30af34..d24f19e6bffa9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -163,7 +163,6 @@ struct VectorizerValueMap {
assert(Instance.Part < UF && "Queried Scalar Part is too large.");
assert(Instance.Lane < VF.getKnownMinValue() &&
"Queried Scalar Lane is too large.");
- assert(!VF.isScalable() && "VF is assumed to be non scalable.");
if (!hasAnyScalarValue(Key))
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/metadata-width.ll b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
index db9c6c9d862ac..e107de672c8be 100644
--- a/llvm/test/Transforms/LoopVectorize/metadata-width.ll
+++ b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
@@ -13,8 +13,7 @@ entry:
for.body: ; preds = %entry, %for.body
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
- %0 = trunc i64 %indvars.iv to i32
- store i32 %0, i32* %arrayidx, align 4
+ store i32 42, i32* %arrayidx, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, %n
@@ -25,7 +24,7 @@ for.end: ; preds = %for.body, %entry
}
; CHECK-LABEL: @test2(
-; CHECK: store <8 x i32>
+; CHECK: store <vscale x 8 x i32>
; CHECK: ret void
define void @test2(i32* nocapture %a, i32 %n) #0 {
entry:
@@ -35,8 +34,7 @@ entry:
for.body: ; preds = %entry, %for.body
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
- %0 = trunc i64 %indvars.iv to i32
- store i32 %0, i32* %arrayidx, align 4
+ store i32 42, i32* %arrayidx, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, %n
@@ -57,8 +55,7 @@ entry:
for.body: ; preds = %entry, %for.body
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
- %0 = trunc i64 %indvars.iv to i32
- store i32 %0, i32* %arrayidx, align 4
+ store i32 42, i32* %arrayidx, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, %n
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
new file mode 100644
index 0000000000000..692b6e5c4ce17
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
@@ -0,0 +1,101 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-interleave=1 < %s | FileCheck %s --check-prefix=CHECKUF1
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-interleave=2 < %s | FileCheck %s --check-prefix=CHECKUF2
+
+; CHECKUF1: for.body.preheader:
+; CHECKUF1-DAG: %wide.trip.count = zext i32 %N to i64
+; CHECKUF1-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX4]], %wide.trip.count
+
+; CHECKUF1: vector.ph:
+; CHECKUF1-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1-DAG: %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX4]]
+; CHECKUF1: %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
+
+; CHECKUF1: vector.body:
+; CHECKUF1: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECKUF1: %[[IDXB:.*]] = getelementptr inbounds double, double* %b, i64 %index
+; CHECKUF1: %[[IDXB_CAST:.*]] = bitcast double* %[[IDXB]] to <vscale x 4 x double>*
+; CHECKUF1: %wide.load = load <vscale x 4 x double>, <vscale x 4 x double>* %[[IDXB_CAST]], align 8, !alias.scope !0
+; CHECKUF1: %[[FADD:.*]] = fadd <vscale x 4 x double> %wide.load, shufflevector (<vscale x 4 x double> insertelement (<vscale x 4 x double> undef, double 1.000000e+00, i32 0), <vscale x 4 x double> undef, <vscale x 4 x i32> zeroinitializer)
+; CHECKUF1: %[[IDXA:.*]] = getelementptr inbounds double, double* %a, i64 %index
+; CHECKUF1: %[[IDXA_CAST:.*]] = bitcast double* %[[IDXA]] to <vscale x 4 x double>*
+; CHECKUF1: store <vscale x 4 x double> %[[FADD]], <vscale x 4 x double>* %[[IDXA_CAST]], align 8, !alias.scope !3, !noalias !0
+; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1: %index.next = add i64 %index, %[[VSCALEX4]]
+; CHECKUF1: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec
+; CHECKUF1: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !5
+
+
+; For an interleave factor of 2, vscale is scaled by 8 instead of 4 (and thus shifted left by 3 instead of 2).
+; There is also the increment for the next iteration, e.g. instead of indexing IDXB, it indexes at IDXB + vscale * 4.
+
+; CHECKUF2: for.body.preheader:
+; CHECKUF2-DAG: %wide.trip.count = zext i32 %N to i64
+; CHECKUF2-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX8]], %wide.trip.count
+
+; CHECKUF2: vector.ph:
+; CHECKUF2-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2-DAG: %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX8]]
+; CHECKUF2: %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
+
+; CHECKUF2: vector.body:
+; CHECKUF2: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECKUF2: %[[IDXB:.*]] = getelementptr inbounds double, double* %b, i64 %index
+; CHECKUF2: %[[IDXB_CAST:.*]] = bitcast double* %[[IDXB]] to <vscale x 4 x double>*
+; CHECKUF2: %wide.load = load <vscale x 4 x double>, <vscale x 4 x double>* %[[IDXB_CAST]], align 8, !alias.scope !0
+; CHECKUF2: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; CHECKUF2: %[[VSCALE2:.*]] = shl i32 %[[VSCALE]], 2
+; CHECKUF2: %[[VSCALE2_EXT:.*]] = sext i32 %[[VSCALE2]] to i64
+; CHECKUF2: %[[IDXB_NEXT:.*]] = getelementptr inbounds double, double* %[[IDXB]], i64 %[[VSCALE2_EXT]]
+; CHECKUF2: %[[IDXB_NEXT_CAST:.*]] = bitcast double* %[[IDXB_NEXT]] to <vscale x 4 x double>*
+; CHECKUF2: %wide.load{{[0-9]+}} = load <vscale x 4 x double>, <vscale x 4 x double>* %[[IDXB_NEXT_CAST]], align 8, !alias.scope !0
+; CHECKUF2: %[[FADD:.*]] = fadd <vscale x 4 x double> %wide.load, shufflevector (<vscale x 4 x double> insertelement (<vscale x 4 x double> undef, double 1.000000e+00, i32 0), <vscale x 4 x double> undef, <vscale x 4 x i32> zeroinitializer)
+; CHECKUF2: %[[FADD_NEXT:.*]] = fadd <vscale x 4 x double> %wide.load{{[0-9]+}}, shufflevector (<vscale x 4 x double> insertelement (<vscale x 4 x double> undef, double 1.000000e+00, i32 0), <vscale x 4 x double> undef, <vscale x 4 x i32> zeroinitializer)
+; CHECKUF2: %[[IDXA:.*]] = getelementptr inbounds double, double* %a, i64 %index
+; CHECKUF2: %[[IDXA_CAST:.*]] = bitcast double* %[[IDXA]] to <vscale x 4 x double>*
+; CHECKUF2: store <vscale x 4 x double> %[[FADD]], <vscale x 4 x double>* %[[IDXA_CAST]], align 8, !alias.scope !3, !noalias !0
+; CHECKUF2: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; CHECKUF2: %[[VSCALE2:.*]] = shl i32 %[[VSCALE]], 2
+; CHECKUF2: %[[VSCALE2_EXT:.*]] = sext i32 %[[VSCALE2]] to i64
+; CHECKUF2: %[[IDXA_NEXT:.*]] = getelementptr inbounds double, double* %[[IDXA]], i64 %[[VSCALE2_EXT]]
+; CHECKUF2: %[[IDXA_NEXT_CAST:.*]] = bitcast double* %[[IDXA_NEXT]] to <vscale x 4 x double>*
+; CHECKUF2: store <vscale x 4 x double> %[[FADD_NEXT]], <vscale x 4 x double>* %[[IDXA_NEXT_CAST]], align 8, !alias.scope !3, !noalias !0
+; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2: %index.next = add i64 %index, %[[VSCALEX8]]
+; CHECKUF2: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec
+; CHECKUF2: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !5
+
+define void @loop(i32 %N, double* nocapture %a, double* nocapture readonly %b) {
+entry:
+ %cmp7 = icmp sgt i32 %N, 0
+ br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ %wide.trip.count = zext i32 %N to i64
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+ %0 = load double, double* %arrayidx, align 8
+ %add = fadd double %0, 1.000000e+00
+ %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+ store double %add, double* %arrayidx2, align 8
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
More information about the llvm-branch-commits
mailing list