[llvm-branch-commits] [llvm] [LV][REVEC] Initial support for re-vectorisation (PR #208213)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 7 07:51:03 PDT 2026
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/208213 at github.com>
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Gaëtan Bossu (gbossu)
<details>
<summary>Changes</summary>
This represent this first piece to enable basic re-vectorisation support in LoopVectorizer. Most of the diff comes from the added tests, the changes to LoopVectorizer files are rather minimal. This proof-of-concept has obvious limitations and only represents the first building block.
Support for re-vectorisation is hidden behind a -vectorize-vector-loops flag and LV will bail out if it encounters constructs that are not yet supported. For example:
- shufflevectors
- gather/scatter and interleaved accesses
- target intrinsics
- reductions
- if-conversion or tail folding
Note: Re-vectorisation was discussed in [[RFC] Re-vectorisation to wider vectors in LoopVectorizer](https://discourse.llvm.org/t/rfc-re-vectorisation-to-wider-vectors-in-loopvectorizer/91071)
---
Patch is 87.00 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/208213.diff
22 Files Affected:
- (modified) llvm/include/llvm/IR/VectorTypeUtils.h (+23-2)
- (modified) llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h (+4)
- (modified) llvm/lib/IR/IRBuilder.cpp (+20)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+58-9)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp (+22-9)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+11-4)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.cpp (+22-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+2-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanHelpers.h (+7)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+35-16)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/revec-maximize-bandwidth.ll (+50)
- (added) llvm/test/Transforms/LoopVectorize/revec-disabled.ll (+43)
- (added) llvm/test/Transforms/LoopVectorize/revec-invalid.ll (+73)
- (added) llvm/test/Transforms/LoopVectorize/revec-livein.ll (+51)
- (added) llvm/test/Transforms/LoopVectorize/revec-liveout.ll (+77)
- (added) llvm/test/Transforms/LoopVectorize/revec-maximize-bandwidth.ll (+45)
- (added) llvm/test/Transforms/LoopVectorize/revec-memory-contiguous.ll (+90)
- (added) llvm/test/Transforms/LoopVectorize/revec-memory-gather-scatter.ll (+119)
- (added) llvm/test/Transforms/LoopVectorize/revec-memory-interleaved.ll (+90)
- (added) llvm/test/Transforms/LoopVectorize/revec-predication.ll (+103)
- (added) llvm/test/Transforms/LoopVectorize/revec-select.ll (+293)
- (added) llvm/test/Transforms/LoopVectorize/revec-unroll.ll (+100)
``````````diff
diff --git a/llvm/include/llvm/IR/VectorTypeUtils.h b/llvm/include/llvm/IR/VectorTypeUtils.h
index e3d7fadad6089..05809f9e07233 100644
--- a/llvm/include/llvm/IR/VectorTypeUtils.h
+++ b/llvm/include/llvm/IR/VectorTypeUtils.h
@@ -14,12 +14,25 @@
namespace llvm {
-/// A helper function for converting Scalar types to vector types. If
+/// A helper function for converting scalar or vector types to vector types. If
/// the incoming type is void, we return void. If the EC represents a
-/// scalar, we return the scalar type.
+/// scalar, we return the input type. For vector inputs, the existing vector
+/// element count is multiplied by EC.
inline Type *toVectorTy(Type *Scalar, ElementCount EC) {
if (Scalar->isVoidTy() || Scalar->isMetadataTy() || EC.isScalar())
return Scalar;
+ if (auto *VTy = dyn_cast<VectorType>(Scalar)) {
+ assert(!(VTy->getElementCount().isScalable() && EC.isScalable()) &&
+ "Attempt to create <vscale x vscale x N x elt>!");
+
+ if (auto *FVTy = dyn_cast<FixedVectorType>(VTy))
+ return VectorType::get(VTy->getElementType(),
+ EC * FVTy->getNumElements());
+
+ return VectorType::get(VTy->getElementType(),
+ VTy->getElementCount() * EC.getKnownMinValue());
+ }
+
return VectorType::get(Scalar, EC);
}
@@ -27,6 +40,14 @@ inline Type *toVectorTy(Type *Scalar, unsigned VF) {
return toVectorTy(Scalar, ElementCount::getFixed(VF));
}
+/// Returns the ElementCount if Ty is a vector type, and 1 otherwise.
+inline ElementCount getElementCount(Type *Ty) {
+ // TODO: Support vectorized structs?
+ if (auto *VTy = dyn_cast<VectorType>(Ty))
+ return VTy->getElementCount();
+ return ElementCount::getFixed(1);
+}
+
/// A helper for converting structs of scalar types to structs of vector types.
/// Note:
/// - If \p EC is scalar, \p StructTy is returned unchanged
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 502fec920ece9..3a114e99ea63f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -48,6 +48,8 @@ class TargetLibraryInfo;
class TargetTransformInfo;
class Type;
+extern cl::opt<bool> VectorizeVectorLoops;
+
/// Utility class for getting and setting loop vectorizer hints in the form
/// of loop metadata.
/// This class keeps a number of loop annotations locally (as member variables)
@@ -495,6 +497,8 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
+ bool LoopContainsVectors = false;
+
private:
/// Return true if the pre-header, exiting and latch blocks of \p Lp and all
/// its nested loops are considered legal for vectorization. These legal
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index df738faad7842..644c9f38bbbae 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -30,6 +30,7 @@
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
+#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/Support/Casting.h"
#include <cassert>
#include <cstdint>
@@ -1286,6 +1287,25 @@ Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
const Twine &Name) {
assert(EC.isNonZero() && "Cannot splat to an empty vector!");
+ if (V->getType()->isVectorTy()) {
+ auto *VectorTy = cast<VectorType>(V->getType());
+ assert(!isa<ScalableVectorType>(VectorTy));
+
+ // If the value was already a constant splat, just recreate it with a
+ // bigger element count.
+ if (auto *VectorConstant = dyn_cast<Constant>(V);
+ VectorConstant && VectorConstant->getSplatValue()) {
+ ElementCount ActualEC =
+ EC.multiplyCoefficientBy(VectorTy->getElementCount().getFixedValue());
+ return CreateVectorSplat(ActualEC, VectorConstant->getSplatValue(), Name);
+ }
+
+ // Otherwise, use vector.broadcast.
+ auto *WideVectorTy = toVectorTy(VectorTy, EC);
+ return CreateIntrinsic(Intrinsic::vector_broadcast,
+ {WideVectorTy, VectorTy}, {V}, {}, Name);
+ }
+
// First insert it into a poison vector so we can shuffle it.
Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index c2c6ac7953d82..88b72fe4728c5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -79,6 +79,13 @@ static cl::opt<bool> EnableHistogramVectorization(
"enable-histogram-loop-vectorization", cl::init(false), cl::Hidden,
cl::desc("Enables autovectorization of some loops containing histograms"));
+namespace llvm {
+cl::opt<bool>
+ VectorizeVectorLoops("vectorize-vector-loops", cl::init(false), cl::Hidden,
+ cl::desc("Allow vectorization of loops with vector "
+ "instructions."));
+} // namespace llvm
+
/// Maximum vectorization interleave count.
static const unsigned MaxInterleaveFactor = 16;
@@ -962,8 +969,18 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (CI && !VFDatabase::getMappings(*CI).empty())
VecCallVariantsFound = true;
+ // REVEC: Remember that a vector instruction was found for later checks.
+ if (I.getType()->isVectorTy() ||
+ any_of(I.operand_values(),
+ [](const Value *V) { return V->getType()->isVectorTy(); }))
+ LoopContainsVectors = true;
+
auto CanWidenInstructionTy = [](Instruction const &Inst) {
Type *InstTy = Inst.getType();
+
+ if (isa<FixedVectorType>(InstTy))
+ return VectorizeVectorLoops.getValue();
+
if (!isa<StructType>(InstTy))
return canVectorizeTy(InstTy);
@@ -974,13 +991,19 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
all_of(Inst.users(), IsaPred<ExtractValueInst>);
};
+ auto CanWidenCast = [&](const Instruction &CastI) {
+ assert(isa<CastInst>(CastI));
+ assert(CanWidenInstructionTy(CastI) &&
+ "CanWidenInstructionTy was not checked beforehand.");
+ Type *FromTy = CastI.getOperand(0)->getType();
+ return VectorType::isValidElementType(FromTy) ||
+ (isa<FixedVectorType>(FromTy) && VectorizeVectorLoops);
+ };
+
// Check that the instruction return type is vectorizable.
- // We can't vectorize casts from vector type to scalar type.
- // Also, we can't vectorize extractelement instructions.
- if (!CanWidenInstructionTy(I) ||
- (isa<CastInst>(I) &&
- !VectorType::isValidElementType(I.getOperand(0)->getType())) ||
- isa<ExtractElementInst>(I)) {
+ // Also, we cannot re-vectorize element or shuffle operations yet.
+ if (!CanWidenInstructionTy(I) || (isa<CastInst>(I) && !CanWidenCast(I)) ||
+ isa<ExtractElementInst, InsertElementInst, ShuffleVectorInst>(I)) {
reportVectorizationFailure("Found unvectorizable type",
"instruction return type cannot be vectorized",
"CantVectorizeInstructionReturnType", ORE,
@@ -991,7 +1014,9 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// Check that the stored type is vectorizable.
if (auto *ST = dyn_cast<StoreInst>(&I)) {
Type *T = ST->getValueOperand()->getType();
- if (!VectorType::isValidElementType(T)) {
+ bool CanWidenStoreType = VectorType::isValidElementType(T) ||
+ (isa<FixedVectorType>(T) && VectorizeVectorLoops);
+ if (!CanWidenStoreType) {
reportVectorizationFailure("Store instruction cannot be vectorized",
"CantVectorizeStore", ORE, TheLoop, ST);
return false;
@@ -1001,7 +1026,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// supported on the target.
if (ST->getMetadata(LLVMContext::MD_nontemporal)) {
// Arbitrarily try a vector of 2 elements.
- auto *VecTy = FixedVectorType::get(T, /*NumElts=*/2);
+ Type *VecTy =
+ T->isVectorTy() ? T : FixedVectorType::get(T, /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of stored type");
if (!TTI->isLegalNTStore(VecTy, ST->getAlign())) {
reportVectorizationFailure(
@@ -1015,7 +1041,9 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (LD->getMetadata(LLVMContext::MD_nontemporal)) {
// For nontemporal loads, check that a nontemporal vector version is
// supported on the target (arbitrarily try a vector of 2 elements).
- auto *VecTy = FixedVectorType::get(I.getType(), /*NumElts=*/2);
+ Type *VecTy = I.getType()->isVectorTy()
+ ? I.getType()
+ : FixedVectorType::get(I.getType(), /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of load type");
if (!TTI->isLegalNTLoad(VecTy, LD->getAlign())) {
reportVectorizationFailure(
@@ -1921,6 +1949,19 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
+ if (LoopContainsVectors && any_of(TheLoop->blocks(), [this](BasicBlock *BB) {
+ return blockNeedsPredication(BB);
+ })) {
+ reportVectorizationFailure("Cannot if-convert vector loop",
+ "if-conversion is not supported for vector "
+ "instructions in loop",
+ "UnsupportedVectorInstruction", ORE, TheLoop);
+ if (DoExtraAnalysis)
+ Result = false;
+ else
+ return false;
+ }
+
if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
if (TheLoop->getExitingBlock()) {
reportVectorizationFailure("Cannot vectorize uncountable loop",
@@ -1990,6 +2031,14 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
+ // TODO-REVEC: Disable tail-folding for now. New intrinsics are needed for
+ // per-segment predication because the element count of the predicate and the
+ // data type do not match.
+ if (LoopContainsVectors) {
+ LLVM_DEBUG(dbgs() << "LV: Tail-folding disabled for REVEC.\n");
+ return false;
+ }
+
// The list of pointers that we can safely read and write to remains empty.
SmallPtrSet<Value *, 8> SafePointers;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index fd7fd2e011a83..7485b7087d4a0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -150,8 +150,11 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
return false;
auto *Ty = getLoadStoreType(V);
Align Align = getLoadStoreAlignment(V);
+ // TODO-REVEC: Support non-contiguous accesses
+ if (Ty->isVectorTy())
+ return false;
if (VF.isVector())
- Ty = VectorType::get(Ty, VF);
+ Ty = toVectorTy(Ty, VF);
return ForceTargetSupportsGatherScatterOps ||
(LI && TTI.isLegalMaskedGather(Ty, Align)) ||
(SI && TTI.isLegalMaskedScatter(Ty, Align));
@@ -346,7 +349,8 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
// Disable scalable vectorization if the loop contains any instructions
// with element types not supported for scalable vectors.
if (any_of(ElementTypesInLoop, [&](Type *Ty) {
- return !Ty->isVoidTy() && !TTI.isElementTypeLegalForScalableVector(Ty);
+ return !Ty->isVoidTy() &&
+ !TTI.isElementTypeLegalForScalableVector(Ty->getScalarType());
})) {
reportVectorizationInfo("Scalable vectorization is not supported "
"for all element types found in this loop.",
@@ -408,6 +412,11 @@ FixedScalableVFPair VFSelectionContext::computeFeasibleMaxVF(
auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElementsPowerOf2);
auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
+ // A vector loop can only be widened to a scalable vector loop for now.
+ // TODO-REVEC: Support fixed-length REVEC.
+ if (Legal->LoopContainsVectors)
+ MaxSafeFixedVF = ElementCount::getFixed(1);
+
if (!Legal->isSafeForAnyVectorWidth())
MaxSafeElements = MaxSafeElementsPowerOf2;
@@ -423,9 +432,13 @@ FixedScalableVFPair VFSelectionContext::computeFeasibleMaxVF(
if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
// If `VF=vscale x N` is safe, then so is `VF=N`
- if (UserVF.isScalable())
- return FixedScalableVFPair(
- ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
+ // (unless it's wider than MaxSafeFixedVF).
+ if (UserVF.isScalable()) {
+ auto UserVFAsFixed = ElementCount::getFixed(UserVF.getKnownMinValue());
+ return ElementCount::isKnownLE(UserVFAsFixed, MaxSafeFixedVF)
+ ? FixedScalableVFPair(UserVFAsFixed, UserVF)
+ : FixedScalableVFPair(MaxSafeFixedVF, UserVF);
+ }
return UserVF;
}
@@ -521,10 +534,10 @@ VFSelectionContext::getSmallestAndWidestTypes() const {
}
} else {
for (Type *T : ElementTypesInLoop) {
- MinWidth = std::min<unsigned>(
- MinWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
- MaxWidth = std::max<unsigned>(
- MaxWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
+ MinWidth =
+ std::min<unsigned>(MinWidth, DL.getTypeSizeInBits(T).getFixedValue());
+ MaxWidth =
+ std::max<unsigned>(MaxWidth, DL.getTypeSizeInBits(T).getFixedValue());
}
}
return {MinWidth, MaxWidth};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 28237c7d037d4..a967d08dc546a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4380,6 +4380,9 @@ LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
Instruction *InsertPos = Group->getInsertPos();
Type *ValTy = getLoadStoreType(InsertPos);
+ if (ValTy->isVectorTy())
+ return InstructionCost::getInvalid();
+
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
unsigned AS = getLoadStoreAddressSpace(InsertPos);
@@ -5136,6 +5139,7 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
// fold away. We can generalize this for all operations using the notion
// of neutral elements. (TODO)
if (I->getOpcode() == Instruction::Mul &&
+ PSE.getSE()->isSCEVable(I->getOperand(0)->getType()) &&
((TheLoop->isLoopInvariant(I->getOperand(0)) &&
PSE.getSCEV(I->getOperand(0))->isOne()) ||
(TheLoop->isLoopInvariant(I->getOperand(1)) &&
@@ -5174,8 +5178,12 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
}
case Instruction::Select: {
SelectInst *SI = cast<SelectInst>(I);
- const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
- bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
+ Type *CondTy = SI->getCondition()->getType();
+ bool ScalarCond = false;
+ if (SE->isSCEVable(CondTy)) {
+ const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
+ ScalarCond = SE->isLoopInvariant(CondSCEV, TheLoop);
+ }
const Value *Op0, *Op1;
using namespace llvm::PatternMatch;
@@ -5194,9 +5202,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
I);
}
- Type *CondTy = SI->getCondition()->getType();
if (!ScalarCond)
- CondTy = VectorType::get(CondTy, VF);
+ CondTy = toVectorTy(CondTy, VF);
CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 5b53312c3ebda..e0648fdee8e86 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -301,11 +301,31 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
assert(hasVectorValue(Def));
auto *VecPart = Data.VPV2Vector[Def];
- if (!VecPart->getType()->isVectorTy()) {
+ // If VecPart's type is the same as the initial pre-vectorisation type,
+ // Def hasn't been vectorised. We are done.
+ Type *InitialTy = Def->getScalarType();
+ if (VecPart->getType() == InitialTy) {
assert(Lane.isFirstLane() && "cannot get lane > 0 for scalar");
return VecPart;
}
+
+ // Last case: Extract a lane of type InitialTy from a vectorised value.
// TODO: Cache created scalar values.
+ if (auto *InitVTy = dyn_cast<VectorType>(InitialTy)) {
+ assert(!InitialTy->isScalableTy() && "REVEC: Unexpected scalable vector");
+ unsigned EC = InitVTy->getElementCount().getFixedValue();
+ if (Lane.getKind() == VPLane::Kind::First)
+ return Builder.CreateExtractVector(InitialTy, VecPart,
+ uint64_t(Lane.getKnownLane() * EC));
+
+ // Shift the demanded InitVTy-typed lane into lane 0.
+ unsigned NumLanesFromEnd =
+ VF.getKnownMinValue() - Lane.getOffsetInLastSubvec();
+ auto *ShiftLastSubvec = Builder.CreateVectorSpliceRight(
+ VecPart, PoisonValue::get(VecPart->getType()), NumLanesFromEnd * EC);
+ return Builder.CreateExtractVector(InitialTy, ShiftLastSubvec, uint64_t(0));
+ }
+
Value *LaneV = Lane.getAsRuntimeExpr(Builder, VF);
auto *Extract = Builder.CreateExtractElement(VecPart, LaneV);
// set(Def, Extract, Instance);
@@ -332,7 +352,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
auto GetBroadcastInstrs = [this](Value *V) {
if (VF.isScalar())
return V;
- // Broadcast the scalar into all locations in the vector.
+ // Broadcast the value into all locations in the vector.
Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
return Shuf;
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..cc76225aabd31 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1511,8 +1511,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns true if the recipe only uses the first part of operand \p Op.
bool usesFirstPartOnly(const VPValue *Op) const override;
- /// Returns true if this VPInstruction produces a scalar value from a vector,
+ /// Whether this VPInstruction turns a VFxInitialTy type back into InitialTy,
/// e.g. by performing a reduction or extracting a lane.
+ /// Note InitialTy might already be a vector if re-vectorising.
bool isVectorToScalar() const;
/// Returns true if the recipe produces a single scalar value.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 1c987abc649c8..1369f004f1821 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -163,6 +163,13 @@ class VPLane {
return Lane;
}
+ /// Returns the offset of this lane from the start of the last
+ /// <N x ElTy> subvector.
+ unsigned getOffsetInLastSubvec() const {
+ assert(LaneKind == Kind::ScalableLast);
+ return Lane;
+ }
+
/// Returns an expression describing the lane index that can be used at
/// runtime.
Value *getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ca63d1498316b..c50b3f5062297 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -494,13 +494,14 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
case Instruction::Store:
return Type::getVoidTy(Ctx);
case Instruction::ICmp:
- assert(Op0Ty->isIntOrPtrTy() && "expected integer or pointer operand");
+ assert(Op0Ty->getScalarType()->isIntOrPtrTy() &&
+ "expected integer or pointer o...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/208213
More information about the llvm-branch-commits
mailing list