[llvm-branch-commits] [llvm] [LV][REVEC][AArch64] Proof of concept for re-vectorisation (PR #208213)
Gaëtan Bossu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 7 04:00:06 PDT 2026
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/208213
>From 384387b36744e0554634af3f45a9e4574a735b28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Mon, 6 Jul 2026 10:32:07 +0000
Subject: [PATCH 1/7] [LV][REVEC][AArch64] Proof of concept for
re-vectorisation
This shows the changes required to enable basic re-vectorisation support in LoopVectorizer. Most of the diff comes from the added tests, the changes to LoopVectorizer files are rather minimal. This proof-of-concept has obvious limitations and only represents the first building block.
My hope is that this helps discussions and complements the RFC at https://discourse.llvm.org/t/rfc-re-vectorisation-to-wider-vectors-in-loopvectorizer/91071.
Support for re-vectorisation is hidden behind a -vectorize-vector-loops flag and LV will bail out if it encounters constructs that are not yet supported. For example:
- shufflevectors
- gather/scatter and interleaved accesses
- target intrinsics
- reductions
- if-conversion or tail folding
---
llvm/include/llvm/IR/VectorTypeUtils.h | 25 +-
.../Vectorize/LoopVectorizationLegality.h | 4 +
llvm/lib/IR/IRBuilder.cpp | 20 ++
.../AArch64/AArch64TargetTransformInfo.h | 9 +
.../Vectorize/LoopVectorizationLegality.cpp | 74 ++++-
.../Vectorize/LoopVectorizationPlanner.cpp | 38 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 15 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 24 +-
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 7 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 36 ++-
.../LoopVectorize/AArch64/revec-disabled.ll | 43 +++
.../LoopVectorize/AArch64/revec-invalid.ll | 73 +++++
.../LoopVectorize/AArch64/revec-livein.ll | 51 +++
.../AArch64/revec-memory-contiguous.ll | 90 ++++++
.../AArch64/revec-memory-gather-scatter.ll | 118 +++++++
.../AArch64/revec-memory-interleaved.ll | 89 ++++++
.../AArch64/revec-predication.ll | 102 ++++++
.../LoopVectorize/AArch64/revec-select.ll | 293 ++++++++++++++++++
.../LoopVectorize/AArch64/revec-unroll.ll | 100 ++++++
19 files changed, 1170 insertions(+), 41 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-disabled.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-livein.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-contiguous.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-gather-scatter.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-interleaved.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-unroll.ll
diff --git a/llvm/include/llvm/IR/VectorTypeUtils.h b/llvm/include/llvm/IR/VectorTypeUtils.h
index e3d7fadad6089..05809f9e07233 100644
--- a/llvm/include/llvm/IR/VectorTypeUtils.h
+++ b/llvm/include/llvm/IR/VectorTypeUtils.h
@@ -14,12 +14,25 @@
namespace llvm {
-/// A helper function for converting Scalar types to vector types. If
+/// A helper function for converting scalar or vector types to vector types. If
/// the incoming type is void, we return void. If the EC represents a
-/// scalar, we return the scalar type.
+/// scalar, we return the input type. For vector inputs, the existing vector
+/// element count is multiplied by EC.
inline Type *toVectorTy(Type *Scalar, ElementCount EC) {
if (Scalar->isVoidTy() || Scalar->isMetadataTy() || EC.isScalar())
return Scalar;
+ if (auto *VTy = dyn_cast<VectorType>(Scalar)) {
+ assert(!(VTy->getElementCount().isScalable() && EC.isScalable()) &&
+ "Attempt to create <vscale x vscale x N x elt>!");
+
+ if (auto *FVTy = dyn_cast<FixedVectorType>(VTy))
+ return VectorType::get(VTy->getElementType(),
+ EC * FVTy->getNumElements());
+
+ return VectorType::get(VTy->getElementType(),
+ VTy->getElementCount() * EC.getKnownMinValue());
+ }
+
return VectorType::get(Scalar, EC);
}
@@ -27,6 +40,14 @@ inline Type *toVectorTy(Type *Scalar, unsigned VF) {
return toVectorTy(Scalar, ElementCount::getFixed(VF));
}
+/// Returns the ElementCount if Ty is a vector type, and 1 otherwise.
+inline ElementCount getElementCount(Type *Ty) {
+ // TODO: Support vectorized structs?
+ if (auto *VTy = dyn_cast<VectorType>(Ty))
+ return VTy->getElementCount();
+ return ElementCount::getFixed(1);
+}
+
/// A helper for converting structs of scalar types to structs of vector types.
/// Note:
/// - If \p EC is scalar, \p StructTy is returned unchanged
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 502fec920ece9..3a114e99ea63f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -48,6 +48,8 @@ class TargetLibraryInfo;
class TargetTransformInfo;
class Type;
+extern cl::opt<bool> VectorizeVectorLoops;
+
/// Utility class for getting and setting loop vectorizer hints in the form
/// of loop metadata.
/// This class keeps a number of loop annotations locally (as member variables)
@@ -495,6 +497,8 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
+ bool LoopContainsVectors = false;
+
private:
/// Return true if the pre-header, exiting and latch blocks of \p Lp and all
/// its nested loops are considered legal for vectorization. These legal
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index df738faad7842..644c9f38bbbae 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -30,6 +30,7 @@
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
+#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/Support/Casting.h"
#include <cassert>
#include <cstdint>
@@ -1286,6 +1287,25 @@ Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
const Twine &Name) {
assert(EC.isNonZero() && "Cannot splat to an empty vector!");
+ if (V->getType()->isVectorTy()) {
+ auto *VectorTy = cast<VectorType>(V->getType());
+ assert(!isa<ScalableVectorType>(VectorTy));
+
+ // If the value was already a constant splat, just recreate it with a
+ // bigger element count.
+ if (auto *VectorConstant = dyn_cast<Constant>(V);
+ VectorConstant && VectorConstant->getSplatValue()) {
+ ElementCount ActualEC =
+ EC.multiplyCoefficientBy(VectorTy->getElementCount().getFixedValue());
+ return CreateVectorSplat(ActualEC, VectorConstant->getSplatValue(), Name);
+ }
+
+ // Otherwise, use vector.broadcast.
+ auto *WideVectorTy = toVectorTy(VectorTy, EC);
+ return CreateIntrinsic(Intrinsic::vector_broadcast,
+ {WideVectorTy, VectorTy}, {V}, {}, Name);
+ }
+
// First insert it into a poison vector so we can shuffle it.
Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ddca341bad05b..1fb39de0c3ee7 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -305,6 +305,15 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
Ty->isIntegerTy(32) || Ty->isIntegerTy(64))
return true;
+ // REVEC: Allow 64-bit and 128-bit vectors, as well as i1 vectors. We still
+ // avoid creating nxv1 types though, as codegen for those can be suboptimal.
+ if (auto *FVTy = dyn_cast<FixedVectorType>(Ty))
+ return (FVTy->getElementType()->isIntegerTy(1) ||
+ DL.getTypeSizeInBits(FVTy) == AArch64::SVEBitsPerBlock ||
+ DL.getTypeSizeInBits(FVTy) == AArch64::SVEBitsPerBlock / 2) &&
+ FVTy->getNumElements() > 1 &&
+ isElementTypeLegalForScalableVector(FVTy->getElementType());
+
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index c2c6ac7953d82..2e94484bb3dca 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -79,6 +79,13 @@ static cl::opt<bool> EnableHistogramVectorization(
"enable-histogram-loop-vectorization", cl::init(false), cl::Hidden,
cl::desc("Enables autovectorization of some loops containing histograms"));
+namespace llvm {
+cl::opt<bool>
+ VectorizeVectorLoops("vectorize-vector-loops", cl::init(false), cl::Hidden,
+ cl::desc("Allow vectorization of loops with vector "
+ "instructions."));
+} // namespace llvm
+
/// Maximum vectorization interleave count.
static const unsigned MaxInterleaveFactor = 16;
@@ -962,8 +969,20 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (CI && !VFDatabase::getMappings(*CI).empty())
VecCallVariantsFound = true;
- auto CanWidenInstructionTy = [](Instruction const &Inst) {
+ // REVEC: Remember that a vector instruction was found for later checks.
+ if (I.getType()->isVectorTy() ||
+ any_of(I.operand_values(),
+ [](const Value *V) { return V->getType()->isVectorTy(); }))
+ LoopContainsVectors = true;
+
+ auto CanWidenInstructionTy = [TTI = TTI](Instruction const &Inst) {
Type *InstTy = Inst.getType();
+
+ // TODO-REVEC: To support fixed VFs, we'll need to query a diffent TTI hook.
+ if (isa<FixedVectorType>(InstTy))
+ return VectorizeVectorLoops &&
+ TTI->isElementTypeLegalForScalableVector(InstTy->getScalarType());
+
if (!isa<StructType>(InstTy))
return canVectorizeTy(InstTy);
@@ -974,13 +993,20 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
all_of(Inst.users(), IsaPred<ExtractValueInst>);
};
+ auto CanWidenCast = [&](const Instruction &CastI) {
+ assert(isa<CastInst>(CastI));
+ assert(CanWidenInstructionTy(CastI) &&
+ "CanWidenInstructionTy was not checked beforehand.");
+ Type *FromTy = CastI.getOperand(0)->getType();
+ return VectorType::isValidElementType(FromTy) ||
+ (isa<FixedVectorType>(FromTy) && VectorizeVectorLoops &&
+ TTI->isElementTypeLegalForScalableVector(FromTy->getScalarType()));
+ };
+
// Check that the instruction return type is vectorizable.
- // We can't vectorize casts from vector type to scalar type.
- // Also, we can't vectorize extractelement instructions.
- if (!CanWidenInstructionTy(I) ||
- (isa<CastInst>(I) &&
- !VectorType::isValidElementType(I.getOperand(0)->getType())) ||
- isa<ExtractElementInst>(I)) {
+ // Also, we cannot re-vectorize element or shuffle operations yet.
+ if (!CanWidenInstructionTy(I) || (isa<CastInst>(I) && !CanWidenCast(I)) ||
+ isa<ExtractElementInst, InsertElementInst, ShuffleVectorInst>(I)) {
reportVectorizationFailure("Found unvectorizable type",
"instruction return type cannot be vectorized",
"CantVectorizeInstructionReturnType", ORE,
@@ -991,7 +1017,11 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// Check that the stored type is vectorizable.
if (auto *ST = dyn_cast<StoreInst>(&I)) {
Type *T = ST->getValueOperand()->getType();
- if (!VectorType::isValidElementType(T)) {
+ bool CanWidenStoreType =
+ VectorType::isValidElementType(T) ||
+ (isa<FixedVectorType>(T) && VectorizeVectorLoops &&
+ TTI->isElementTypeLegalForScalableVector(T->getScalarType()));
+ if (!CanWidenStoreType) {
reportVectorizationFailure("Store instruction cannot be vectorized",
"CantVectorizeStore", ORE, TheLoop, ST);
return false;
@@ -1001,7 +1031,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// supported on the target.
if (ST->getMetadata(LLVMContext::MD_nontemporal)) {
// Arbitrarily try a vector of 2 elements.
- auto *VecTy = FixedVectorType::get(T, /*NumElts=*/2);
+ Type *VecTy =
+ T->isVectorTy() ? T : FixedVectorType::get(T, /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of stored type");
if (!TTI->isLegalNTStore(VecTy, ST->getAlign())) {
reportVectorizationFailure(
@@ -1015,7 +1046,9 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (LD->getMetadata(LLVMContext::MD_nontemporal)) {
// For nontemporal loads, check that a nontemporal vector version is
// supported on the target (arbitrarily try a vector of 2 elements).
- auto *VecTy = FixedVectorType::get(I.getType(), /*NumElts=*/2);
+ Type *VecTy = I.getType()->isVectorTy()
+ ? I.getType()
+ : FixedVectorType::get(I.getType(), /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of load type");
if (!TTI->isLegalNTLoad(VecTy, LD->getAlign())) {
reportVectorizationFailure(
@@ -1921,6 +1954,19 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
+ if (LoopContainsVectors && any_of(TheLoop->blocks(), [this](BasicBlock *BB) {
+ return blockNeedsPredication(BB);
+ })) {
+ reportVectorizationFailure("Cannot if-convert vector loop",
+ "if-conversion is not supported for vector "
+ "instructions in loop",
+ "UnsupportedVectorInstruction", ORE, TheLoop);
+ if (DoExtraAnalysis)
+ Result = false;
+ else
+ return false;
+ }
+
if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
if (TheLoop->getExitingBlock()) {
reportVectorizationFailure("Cannot vectorize uncountable loop",
@@ -1990,6 +2036,14 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
+ // TODO-REVEC: Disable tail-folding for now. New intrinsics are needed for
+ // per-segment predication because the element count of the predicate and the
+ // data type do not match.
+ if (LoopContainsVectors) {
+ LLVM_DEBUG(dbgs() << "LV: Tail-folding disabled for REVEC.\n");
+ return false;
+ }
+
// The list of pointers that we can safely read and write to remains empty.
SmallPtrSet<Value *, 8> SafePointers;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index fd7fd2e011a83..31b0f879a6925 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -150,8 +150,11 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
return false;
auto *Ty = getLoadStoreType(V);
Align Align = getLoadStoreAlignment(V);
+ // TODO-REVEC: Support non-contiguous accesses
+ if (Ty->isVectorTy())
+ return false;
if (VF.isVector())
- Ty = VectorType::get(Ty, VF);
+ Ty = toVectorTy(Ty, VF);
return ForceTargetSupportsGatherScatterOps ||
(LI && TTI.isLegalMaskedGather(Ty, Align)) ||
(SI && TTI.isLegalMaskedScatter(Ty, Align));
@@ -275,7 +278,9 @@ ElementCount VFSelectionContext::getMaximizedVFForTarget(
else
MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
- if (useMaxBandwidth(ComputeScalableMaxVF)) {
+ // REVEC: Avoid creating wider than expected scalable types by choosing
+ // VF > vscale x 1.
+ if (useMaxBandwidth(ComputeScalableMaxVF) && !Legal->LoopContainsVectors) {
auto MaxVectorElementCountMaxBW = ElementCount::get(
llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
ComputeScalableMaxVF);
@@ -346,6 +351,12 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
// Disable scalable vectorization if the loop contains any instructions
// with element types not supported for scalable vectors.
if (any_of(ElementTypesInLoop, [&](Type *Ty) {
+ if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
+ if (!VectorizeVectorLoops)
+ return true;
+ Ty = FVTy->getElementType();
+ } else if (Ty->isVectorTy())
+ return true;
return !Ty->isVoidTy() && !TTI.isElementTypeLegalForScalableVector(Ty);
})) {
reportVectorizationInfo("Scalable vectorization is not supported "
@@ -408,6 +419,11 @@ FixedScalableVFPair VFSelectionContext::computeFeasibleMaxVF(
auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElementsPowerOf2);
auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
+ // A vector loop can only be widened to a scalable vector loop for now.
+ // TODO-REVEC: Support fixed-length REVEC.
+ if (Legal->LoopContainsVectors)
+ MaxSafeFixedVF = ElementCount::getFixed(1);
+
if (!Legal->isSafeForAnyVectorWidth())
MaxSafeElements = MaxSafeElementsPowerOf2;
@@ -423,9 +439,13 @@ FixedScalableVFPair VFSelectionContext::computeFeasibleMaxVF(
if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
// If `VF=vscale x N` is safe, then so is `VF=N`
- if (UserVF.isScalable())
- return FixedScalableVFPair(
- ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
+ // (unless it's wider than MaxSafeFixedVF).
+ if (UserVF.isScalable()) {
+ auto UserVFAsFixed = ElementCount::getFixed(UserVF.getKnownMinValue());
+ return ElementCount::isKnownLE(UserVFAsFixed, MaxSafeFixedVF)
+ ? FixedScalableVFPair(UserVFAsFixed, UserVF)
+ : FixedScalableVFPair(MaxSafeFixedVF, UserVF);
+ }
return UserVF;
}
@@ -521,10 +541,10 @@ VFSelectionContext::getSmallestAndWidestTypes() const {
}
} else {
for (Type *T : ElementTypesInLoop) {
- MinWidth = std::min<unsigned>(
- MinWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
- MaxWidth = std::max<unsigned>(
- MaxWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
+ MinWidth =
+ std::min<unsigned>(MinWidth, DL.getTypeSizeInBits(T).getFixedValue());
+ MaxWidth =
+ std::max<unsigned>(MaxWidth, DL.getTypeSizeInBits(T).getFixedValue());
}
}
return {MinWidth, MaxWidth};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 28237c7d037d4..a967d08dc546a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4380,6 +4380,9 @@ LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
Instruction *InsertPos = Group->getInsertPos();
Type *ValTy = getLoadStoreType(InsertPos);
+ if (ValTy->isVectorTy())
+ return InstructionCost::getInvalid();
+
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
unsigned AS = getLoadStoreAddressSpace(InsertPos);
@@ -5136,6 +5139,7 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
// fold away. We can generalize this for all operations using the notion
// of neutral elements. (TODO)
if (I->getOpcode() == Instruction::Mul &&
+ PSE.getSE()->isSCEVable(I->getOperand(0)->getType()) &&
((TheLoop->isLoopInvariant(I->getOperand(0)) &&
PSE.getSCEV(I->getOperand(0))->isOne()) ||
(TheLoop->isLoopInvariant(I->getOperand(1)) &&
@@ -5174,8 +5178,12 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
}
case Instruction::Select: {
SelectInst *SI = cast<SelectInst>(I);
- const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
- bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
+ Type *CondTy = SI->getCondition()->getType();
+ bool ScalarCond = false;
+ if (SE->isSCEVable(CondTy)) {
+ const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
+ ScalarCond = SE->isLoopInvariant(CondSCEV, TheLoop);
+ }
const Value *Op0, *Op1;
using namespace llvm::PatternMatch;
@@ -5194,9 +5202,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
I);
}
- Type *CondTy = SI->getCondition()->getType();
if (!ScalarCond)
- CondTy = VectorType::get(CondTy, VF);
+ CondTy = toVectorTy(CondTy, VF);
CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 5b53312c3ebda..e0648fdee8e86 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -301,11 +301,31 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
assert(hasVectorValue(Def));
auto *VecPart = Data.VPV2Vector[Def];
- if (!VecPart->getType()->isVectorTy()) {
+ // If VecPart's type is the same as the initial pre-vectorisation type,
+ // Def hasn't been vectorised. We are done.
+ Type *InitialTy = Def->getScalarType();
+ if (VecPart->getType() == InitialTy) {
assert(Lane.isFirstLane() && "cannot get lane > 0 for scalar");
return VecPart;
}
+
+ // Last case: Extract a lane of type InitialTy from a vectorised value.
// TODO: Cache created scalar values.
+ if (auto *InitVTy = dyn_cast<VectorType>(InitialTy)) {
+ assert(!InitialTy->isScalableTy() && "REVEC: Unexpected scalable vector");
+ unsigned EC = InitVTy->getElementCount().getFixedValue();
+ if (Lane.getKind() == VPLane::Kind::First)
+ return Builder.CreateExtractVector(InitialTy, VecPart,
+ uint64_t(Lane.getKnownLane() * EC));
+
+ // Shift the demanded InitVTy-typed lane into lane 0.
+ unsigned NumLanesFromEnd =
+ VF.getKnownMinValue() - Lane.getOffsetInLastSubvec();
+ auto *ShiftLastSubvec = Builder.CreateVectorSpliceRight(
+ VecPart, PoisonValue::get(VecPart->getType()), NumLanesFromEnd * EC);
+ return Builder.CreateExtractVector(InitialTy, ShiftLastSubvec, uint64_t(0));
+ }
+
Value *LaneV = Lane.getAsRuntimeExpr(Builder, VF);
auto *Extract = Builder.CreateExtractElement(VecPart, LaneV);
// set(Def, Extract, Instance);
@@ -332,7 +352,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
auto GetBroadcastInstrs = [this](Value *V) {
if (VF.isScalar())
return V;
- // Broadcast the scalar into all locations in the vector.
+ // Broadcast the value into all locations in the vector.
Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
return Shuf;
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 1c987abc649c8..1369f004f1821 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -163,6 +163,13 @@ class VPLane {
return Lane;
}
+ /// Returns the offset of this lane from the start of the last
+ /// <N x ElTy> subvector.
+ unsigned getOffsetInLastSubvec() const {
+ assert(LaneKind == Kind::ScalableLast);
+ return Lane;
+ }
+
/// Returns an expression describing the lane index that can be used at
/// runtime.
Value *getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ca63d1498316b..82a97ac1883f4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -494,13 +494,9 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
case Instruction::Store:
return Type::getVoidTy(Ctx);
case Instruction::ICmp:
- assert(Op0Ty->isIntOrPtrTy() && "expected integer or pointer operand");
- AssertOperandType(1, Op0Ty);
- return IntegerType::get(Ctx, 1);
case Instruction::FCmp:
- assert(Op0Ty->isFloatingPointTy() && "expected floating-point operand");
AssertOperandType(1, Op0Ty);
- return IntegerType::get(Ctx, 1);
+ return CmpInst::makeCmpResultType(Op0Ty);
case VPInstruction::ActiveLaneMask:
assert(Op0Ty->isIntegerTy() && "expected integer operand");
AssertOperandType(1, Op0Ty);
@@ -522,8 +518,6 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
assert(Op0Ty->isIntegerTy() && "expected integer operand");
return IntegerType::get(Ctx, 32);
case Instruction::Select: {
- assert((!Op0Ty || Op0Ty->isIntegerTy(1)) &&
- "select condition must be bool");
Type *Op1Ty = Operands[1]->getScalarType();
AssertOperandType(2, Op1Ty);
return Op1Ty;
@@ -1267,8 +1261,16 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
}
case Instruction::Select: {
SelectInst *SI = cast_or_null<SelectInst>(getUnderlyingValue());
- bool IsScalarCond = getOperand(0)->isDefinedOutsideLoopRegions();
+ Type *InitialCondTy = getOperand(0)->getScalarType();
Type *ScalarTy = this->getScalarType();
+ bool IsScalarCond = InitialCondTy->isIntegerTy(1) &&
+ getOperand(0)->isDefinedOutsideLoopRegions();
+
+ // TODO-REVEC: Support all kinds of InitialCondTy
+ if (ScalarTy->isVectorTy() &&
+ (!IsScalarCond &&
+ getElementCount(ScalarTy) != getElementCount(InitialCondTy)))
+ return InstructionCost::getInvalid();
VPValue *Op0, *Op1;
bool IsLogicalAnd =
@@ -1299,9 +1301,7 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
Ctx.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, Operands, SI);
}
- Type *CondTy = getOperand(0)->getScalarType();
- if (!IsScalarCond && VF.isVector())
- CondTy = VectorType::get(CondTy, VF);
+ Type *CondTy = IsScalarCond ? InitialCondTy : toVectorTy(InitialCondTy, VF);
llvm::CmpPredicate Pred;
if (!match(getOperand(0), m_Cmp(Pred, m_VPValue(), m_VPValue())))
@@ -1340,6 +1340,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
match(getOperand(0), m_Cmp(Pred, m_VPValue(), m_VPValue()));
auto *CondTy = getOperand(0)->getScalarType();
auto *VecTy = getOperand(1)->getScalarType();
+ assert(!CondTy->isVectorTy() &&
+ "REVEC: Unexpected VPlan-created select at this stage");
if (!vputils::onlyFirstLaneUsed(this)) {
CondTy = toVectorTy(CondTy, VF);
VecTy = toVectorTy(VecTy, VF);
@@ -2830,7 +2832,13 @@ void VPWidenRecipe::execute(VPTransformState &State) {
}
case Instruction::Select: {
VPValue *CondOp = getOperand(0);
- Value *Cond = State.get(CondOp, vputils::isSingleScalar(CondOp));
+ Type *InitialCondTy = CondOp->getScalarType();
+ // REVEC: The initial CondOp might be a vector, so be sure we either
+ // generate a single i1 condition, or an i1 vector with the same EC as the
+ // data inputs.
+ bool BuildScalarCond =
+ vputils::isSingleScalar(CondOp) && InitialCondTy->isIntegerTy(1);
+ Value *Cond = State.get(CondOp, BuildScalarCond);
Value *Op0 = State.get(getOperand(1));
Value *Op1 = State.get(getOperand(2));
Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
@@ -2852,7 +2860,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
#if !defined(NDEBUG)
// Verify that VPlan type inference results agree with the type of the
// generated values.
- assert(VectorType::get(this->getScalarType(), State.VF) ==
+ assert(toVectorTy(this->getScalarType(), State.VF) ==
State.get(this)->getType() &&
"inferred type and type from generated instructions do not match");
#endif
@@ -4184,7 +4192,7 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
void VPWidenLoadRecipe::execute(VPTransformState &State) {
Type *ScalarDataTy = getScalarType();
- auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
+ auto *DataTy = toVectorTy(ScalarDataTy, State.VF);
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-disabled.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-disabled.ll
new file mode 100644
index 0000000000000..ee50a9fca7b61
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-disabled.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -force-vector-interleave=1 -vectorize-vector-loops=false \
+; RUN: < %s | FileCheck %s --check-prefix=NO-REVEC
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -force-vector-interleave=2 -vectorize-vector-loops=false \
+; RUN: < %s | FileCheck %s --check-prefix=NO-REVEC
+
+; Make sure vector loops aren't modified when -vectorize-vector-loops is false.
+
+define void @copy_v4(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; NO-REVEC-LABEL: define void @copy_v4(
+; NO-REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-REVEC-NEXT: [[ENTRY:.*]]:
+; NO-REVEC-NEXT: br label %[[LOOP:.*]]
+; NO-REVEC: [[LOOP]]:
+; NO-REVEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-REVEC-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[IV]]
+; NO-REVEC-NEXT: [[DST_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[IV]]
+; NO-REVEC-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[SRC_GEP]], align 16
+; NO-REVEC-NEXT: store <4 x i32> [[V]], ptr [[DST_GEP]], align 16
+; NO-REVEC-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; NO-REVEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-REVEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-REVEC: [[EXIT]]:
+; NO-REVEC-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src.gep = getelementptr inbounds <4 x i32>, ptr %src, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %v = load <4 x i32>, ptr %src.gep, align 16
+ store <4 x i32> %v, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
new file mode 100644
index 0000000000000..8695676c6e49d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
@@ -0,0 +1,73 @@
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s --check-prefix=IR
+; RUN: opt -disable-output -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
+; RUN: -vectorize-vector-loops -pass-remarks-analysis=loop-vectorize \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=REMARKS
+
+; IR-LABEL: @insertelement_cost(
+; IR-NOT: vector.body:
+; REMARKS: loop not vectorized: instruction return type cannot be vectorized
+define void @insertelement_cost(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+entry:
+ br label %for.body
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i16, ptr %b, i64 %indvars.iv
+ %0 = load i16, ptr %arrayidx, align 16
+ %result = insertelement <8 x i16> zeroinitializer, i16 %0, i32 0
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; IR-LABEL: @extractelement_cost(
+; IR-NOT: vector.body:
+; REMARKS: loop not vectorized: instruction return type cannot be vectorized
+define void @extractelement_cost(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+entry:
+ br label %for.body
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %result = extractelement <4 x i32> %0, i32 1
+ %arrayidx2 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; IR-LABEL: @shufflevector_cost(
+; IR-NOT: vector.body:
+; REMARKS: loop not vectorized: instruction return type cannot be vectorized
+define void @shufflevector_cost(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+entry:
+ br label %for.body
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i32> %0, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-livein.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-livein.ll
new file mode 100644
index 0000000000000..20d284bb10579
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-livein.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s
+
+define void @select_livein_v4(ptr noalias %dst, ptr noalias %src, <4 x i32> %threshold, i64 %n) {
+; CHECK-LABEL: define void @select_livein_v4(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], <4 x i32> [[THRESHOLD:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> [[THRESHOLD]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST]]
+; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[BROADCAST]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP4]], ptr [[TMP2]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src.gep = getelementptr inbounds <4 x i32>, ptr %src, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %v = load <4 x i32>, ptr %src.gep, align 16
+ %cmp = icmp sgt <4 x i32> %v, %threshold
+ %sel = select <4 x i1> %cmp, <4 x i32> %v, <4 x i32> %threshold
+ store <4 x i32> %sel, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-contiguous.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-contiguous.ll
new file mode 100644
index 0000000000000..128443f63c079
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-contiguous.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: < %s | FileCheck %s --check-prefix=REVEC
+
+define void @copy_v4(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; REVEC-LABEL: define void @copy_v4(
+; REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; REVEC-NEXT: [[ENTRY:.*:]]
+; REVEC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; REVEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; REVEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; REVEC: [[VECTOR_PH]]:
+; REVEC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; REVEC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; REVEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; REVEC: [[VECTOR_BODY]]:
+; REVEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; REVEC-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[INDEX]]
+; REVEC-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; REVEC-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 16
+; REVEC-NEXT: store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 16
+; REVEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; REVEC-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; REVEC-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; REVEC: [[MIDDLE_BLOCK]]:
+; REVEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; REVEC-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; REVEC: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src.gep = getelementptr inbounds <4 x i32>, ptr %src, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %v = load <4 x i32>, ptr %src.gep, align 16
+ store <4 x i32> %v, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @copy_v4_nontemporal(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; REVEC-LABEL: define void @copy_v4_nontemporal(
+; REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; REVEC-NEXT: [[ENTRY:.*:]]
+; REVEC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; REVEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; REVEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; REVEC: [[VECTOR_PH]]:
+; REVEC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; REVEC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; REVEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; REVEC: [[VECTOR_BODY]]:
+; REVEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; REVEC-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[INDEX]]
+; REVEC-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; REVEC-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 16, !nontemporal [[META4:![0-9]+]]
+; REVEC-NEXT: store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 16, !nontemporal [[META4]]
+; REVEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; REVEC-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; REVEC-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; REVEC: [[MIDDLE_BLOCK]]:
+; REVEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; REVEC-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; REVEC: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src.gep = getelementptr inbounds <4 x i32>, ptr %src, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %v = load <4 x i32>, ptr %src.gep, align 16, !nontemporal !0
+ store <4 x i32> %v, ptr %dst.gep, align 16, !nontemporal !0
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-gather-scatter.ll
new file mode 100644
index 0000000000000..cbb5aa7d8bd0a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-gather-scatter.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s
+
+define void @strided_v4(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @strided_v4(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IDX:%.*]] = shl nuw i64 [[IV]], 1
+; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[IDX]]
+; CHECK-NEXT: [[DST_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[IDX]]
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[SRC_GEP]], align 16
+; CHECK-NEXT: store <4 x i32> [[V]], ptr [[DST_GEP]], align 16
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = shl nuw i64 %iv, 1
+ %src.gep = getelementptr inbounds <4 x i32>, ptr %src, i64 %idx
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %idx
+ %v = load <4 x i32>, ptr %src.gep, align 16
+ store <4 x i32> %v, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @gather_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @gather_128(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[OFFSET]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], splat (i16 1)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[TMP6]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %offset
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+
+define void @scatter_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @scatter_128(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], splat (i16 1)
+; CHECK-NEXT: [[OFFSET:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %offset
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-interleaved.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-interleaved.ll
new file mode 100644
index 0000000000000..a78af8d6fe473
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-interleaved.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s
+
+
+define void @ld2q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld2q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %result = add <8 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @st2q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @st2q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_IN:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = load <8 x i16>, ptr [[ARRAYIDX_IN]], align 16
+; CHECK-NEXT: [[V0:%.*]] = add <8 x i16> [[TMP9]], splat (i16 1)
+; CHECK-NEXT: [[V1:%.*]] = add <8 x i16> [[V0]], splat (i16 1)
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.in = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx.in, align 16
+ %v0 = add <8 x i16> %0, splat (i16 1)
+ %v1 = add <8 x i16> %v0, splat (i16 1)
+
+ %arrayidx.out0 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
new file mode 100644
index 0000000000000..f3a3ba7f00541
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s
+
+define void @predicated_v4(ptr noalias %dst, ptr noalias %src, i64 %n, i64 %limit) {
+; CHECK-LABEL: define void @predicated_v4(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]], i64 [[LIMIT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[PRED:%.*]] = icmp ult i64 [[IV]], [[LIMIT]]
+; CHECK-NEXT: br i1 [[PRED]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[DST_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[SRC_GEP]], align 16
+; CHECK-NEXT: store <4 x i32> [[V]], ptr [[DST_GEP]], align 16
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %pred = icmp ult i64 %iv, %limit
+ br i1 %pred, label %then, label %latch
+
+then:
+ %src.gep = getelementptr inbounds <4 x i32>, ptr %src, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %v = load <4 x i32>, ptr %src.gep, align 16
+ store <4 x i32> %v, ptr %dst.gep, align 16
+ br label %latch
+
+latch:
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @predicated_load_uniform_cond(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, i1 %cond) {
+; CHECK-LABEL: define void @predicated_load_uniform_cond(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[PRED_1:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: br label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[TMP1:%.*]] = phi <8 x i16> [ zeroinitializer, %[[FOR_BODY]] ], [ [[PRED_1]], %[[IF_THEN]] ]
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.inc
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ br i1 %cond, label %if.then, label %for.inc
+
+if.then:
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %c, i64 %indvars.iv
+ %pred.1 = load <8 x i16>, ptr %arrayidx1, align 16
+ br label %for.inc
+
+for.inc:
+ %1 = phi <8 x i16> [ zeroinitializer, %for.body ], [ %pred.1, %if.then ]
+ %result = add <8 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
new file mode 100644
index 0000000000000..2d837a5236e51
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s
+
+define void @icmp_select_v4(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
+; CHECK-LABEL: define void @icmp_select_v4(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP5]], ptr [[TMP3]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %lhs.gep = getelementptr inbounds <4 x i32>, ptr %lhs, i64 %iv
+ %rhs.gep = getelementptr inbounds <4 x i32>, ptr %rhs, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %a = load <4 x i32>, ptr %lhs.gep, align 16
+ %b = load <4 x i32>, ptr %rhs.gep, align 16
+ %cmp = icmp sgt <4 x i32> %a, %b
+ %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
+ store <4 x i32> %sel, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @icmp_select_v4_uniform_cond(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n, <4 x i32> %conds) {
+; CHECK-LABEL: define void @icmp_select_v4_uniform_cond(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]], <4 x i32> [[CONDS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <4 x i32> [[CONDS]], zeroinitializer
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v4i1(<4 x i1> [[CMP]])
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[BROADCAST]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP4]], ptr [[TMP3]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ %cmp = icmp sgt <4 x i32> %conds, splat (i32 0)
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %lhs.gep = getelementptr inbounds <4 x i32>, ptr %lhs, i64 %iv
+ %rhs.gep = getelementptr inbounds <4 x i32>, ptr %rhs, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %a = load <4 x i32>, ptr %lhs.gep, align 16
+ %b = load <4 x i32>, ptr %rhs.gep, align 16
+ %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
+ store <4 x i32> %sel, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @icmp_select_scalar_cond_uniform(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n, i1 %cond) {
+; CHECK-LABEL: define void @icmp_select_scalar_cond_uniform(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[COND]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP4]], ptr [[TMP3]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %lhs.gep = getelementptr inbounds <4 x i32>, ptr %lhs, i64 %iv
+ %rhs.gep = getelementptr inbounds <4 x i32>, ptr %rhs, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %a = load <4 x i32>, ptr %lhs.gep, align 16
+ %b = load <4 x i32>, ptr %rhs.gep, align 16
+ %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+ store <4 x i32> %sel, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; TODO-REVEC: LV does not handle stretching a nxv1i1 predicate to nxv4i1
+define void @icmp_select_scalar_cond_load(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, ptr noalias %cond, i64 %n) {
+; CHECK-LABEL: define void @icmp_select_scalar_cond_load(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LHS_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[IV]]
+; CHECK-NEXT: [[RHS_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[IV]]
+; CHECK-NEXT: [[COND_GEP:%.*]] = getelementptr inbounds i8, ptr [[COND]], i64 [[IV]]
+; CHECK-NEXT: [[DST_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load <4 x i32>, ptr [[LHS_GEP]], align 16
+; CHECK-NEXT: [[B:%.*]] = load <4 x i32>, ptr [[RHS_GEP]], align 16
+; CHECK-NEXT: [[COND_I8:%.*]] = load i8, ptr [[COND_GEP]], align 1
+; CHECK-NEXT: [[COND_I1:%.*]] = icmp ne i8 [[COND_I8]], 0
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND_I1]], <4 x i32> [[A]], <4 x i32> [[B]]
+; CHECK-NEXT: store <4 x i32> [[SEL]], ptr [[DST_GEP]], align 16
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %lhs.gep = getelementptr inbounds <4 x i32>, ptr %lhs, i64 %iv
+ %rhs.gep = getelementptr inbounds <4 x i32>, ptr %rhs, i64 %iv
+ %cond.gep = getelementptr inbounds i8, ptr %cond, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %a = load <4 x i32>, ptr %lhs.gep, align 16
+ %b = load <4 x i32>, ptr %rhs.gep, align 16
+ %cond.i8 = load i8, ptr %cond.gep, align 1
+ %cond.i1 = icmp ne i8 %cond.i8, 0
+ %sel = select i1 %cond.i1, <4 x i32> %a, <4 x i32> %b
+ store <4 x i32> %sel, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; TODO-REVEC: LV does not handle stretching a nxv1i1 predicate to nxv4i1
+define void @icmp_select_scalar_cond_iv(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
+; CHECK-LABEL: define void @icmp_select_scalar_cond_iv(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LHS_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[IV]]
+; CHECK-NEXT: [[RHS_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[IV]]
+; CHECK-NEXT: [[DST_GEP:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load <4 x i32>, ptr [[LHS_GEP]], align 16
+; CHECK-NEXT: [[B:%.*]] = load <4 x i32>, ptr [[RHS_GEP]], align 16
+; CHECK-NEXT: [[COND_I1:%.*]] = icmp ugt i64 [[IV]], 8
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND_I1]], <4 x i32> [[A]], <4 x i32> [[B]]
+; CHECK-NEXT: store <4 x i32> [[SEL]], ptr [[DST_GEP]], align 16
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %lhs.gep = getelementptr inbounds <4 x i32>, ptr %lhs, i64 %iv
+ %rhs.gep = getelementptr inbounds <4 x i32>, ptr %rhs, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %a = load <4 x i32>, ptr %lhs.gep, align 16
+ %b = load <4 x i32>, ptr %rhs.gep, align 16
+ %cond.i1 = icmp ugt i64 %iv, 8
+ %sel = select i1 %cond.i1, <4 x i32> %a, <4 x i32> %b
+ store <4 x i32> %sel, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @fcmp_select_v4(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
+; CHECK-LABEL: define void @fcmp_select_v4(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[LHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[RHS]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP4:%.*]] = fcmp olt <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD1]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP5]], ptr [[TMP3]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %lhs.gep = getelementptr inbounds <4 x float>, ptr %lhs, i64 %iv
+ %rhs.gep = getelementptr inbounds <4 x float>, ptr %rhs, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x float>, ptr %dst, i64 %iv
+ %a = load <4 x float>, ptr %lhs.gep, align 16
+ %b = load <4 x float>, ptr %rhs.gep, align 16
+ %cmp = fcmp olt <4 x float> %a, %b
+ %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %b
+ store <4 x float> %sel, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-unroll.ll
new file mode 100644
index 0000000000000..feaef7eaaf1d3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-unroll.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -force-vector-width=1 -force-vector-interleave=2 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s --check-prefix=VF1IC2
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
+; RUN: -force-vector-width="vscale x 1" -force-vector-interleave=2 \
+; RUN: -vectorize-vector-loops < %s | FileCheck %s --check-prefix=SVF1IC2
+
+; Verify that we can interleave vector loop iterations.
+
+define void @load_add_store(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
+; VF1IC2-LABEL: define void @load_add_store(
+; VF1IC2-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; VF1IC2-NEXT: [[ENTRY:.*:]]
+; VF1IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; VF1IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF1IC2: [[VECTOR_PH]]:
+; VF1IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; VF1IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF1IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF1IC2: [[VECTOR_BODY]]:
+; VF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF1IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; VF1IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[INDEX]]
+; VF1IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[INDEX]]
+; VF1IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; VF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP7:%.*]] = load <4 x i32>, ptr [[TMP1]], align 16
+; VF1IC2-NEXT: [[TMP8:%.*]] = load <4 x i32>, ptr [[TMP2]], align 16
+; VF1IC2-NEXT: [[TMP9:%.*]] = load <4 x i32>, ptr [[TMP3]], align 16
+; VF1IC2-NEXT: [[TMP10:%.*]] = load <4 x i32>, ptr [[TMP4]], align 16
+; VF1IC2-NEXT: [[TMP11:%.*]] = add <4 x i32> [[TMP7]], [[TMP9]]
+; VF1IC2-NEXT: [[TMP12:%.*]] = add <4 x i32> [[TMP8]], [[TMP10]]
+; VF1IC2-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP5]], align 16
+; VF1IC2-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 16
+; VF1IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF1IC2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VF1IC2: [[MIDDLE_BLOCK]]:
+; VF1IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF1IC2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF1IC2: [[SCALAR_PH]]:
+;
+; SVF1IC2-LABEL: define void @load_add_store(
+; SVF1IC2-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; SVF1IC2-NEXT: [[ENTRY:.*:]]
+; SVF1IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; SVF1IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; SVF1IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; SVF1IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SVF1IC2: [[VECTOR_PH]]:
+; SVF1IC2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; SVF1IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; SVF1IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; SVF1IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; SVF1IC2: [[VECTOR_BODY]]:
+; SVF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SVF1IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[LHS]], i64 [[INDEX]]
+; SVF1IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[RHS]], i64 [[INDEX]]
+; SVF1IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[INDEX]]
+; SVF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP3]], i64 [[TMP0]]
+; SVF1IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 16
+; SVF1IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 16
+; SVF1IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP4]], i64 [[TMP0]]
+; SVF1IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 16
+; SVF1IC2-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 16
+; SVF1IC2-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; SVF1IC2-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; SVF1IC2-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP5]], i64 [[TMP0]]
+; SVF1IC2-NEXT: store <vscale x 4 x i32> [[TMP8]], ptr [[TMP5]], align 16
+; SVF1IC2-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 16
+; SVF1IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; SVF1IC2-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SVF1IC2-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; SVF1IC2: [[MIDDLE_BLOCK]]:
+; SVF1IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; SVF1IC2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; SVF1IC2: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %lhs.gep = getelementptr inbounds <4 x i32>, ptr %lhs, i64 %iv
+ %rhs.gep = getelementptr inbounds <4 x i32>, ptr %rhs, i64 %iv
+ %dst.gep = getelementptr inbounds <4 x i32>, ptr %dst, i64 %iv
+ %a = load <4 x i32>, ptr %lhs.gep, align 16
+ %b = load <4 x i32>, ptr %rhs.gep, align 16
+ %add = add <4 x i32> %a, %b
+ store <4 x i32> %add, ptr %dst.gep, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 44640daa97ac9a1d70cd6150399e51bfd133e834 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 4 Aug 2026 12:24:55 +0000
Subject: [PATCH 2/7] Remove aarch64 triple and move tests out of AArch64/
---
.../{AArch64 => }/revec-disabled.ll | 10 +++++-----
.../{AArch64 => }/revec-invalid.ll | 12 ++++++------
.../{AArch64 => }/revec-livein.ll | 8 ++++----
.../{AArch64 => }/revec-memory-contiguous.ll | 8 ++++----
.../revec-memory-gather-scatter.ll | 13 +++++++------
.../{AArch64 => }/revec-memory-interleaved.ll | 11 ++++++-----
.../{AArch64 => }/revec-predication.ll | 11 ++++++-----
.../{AArch64 => }/revec-select.ll | 18 +++++++++---------
.../{AArch64 => }/revec-unroll.ll | 16 ++++++++--------
9 files changed, 55 insertions(+), 52 deletions(-)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-disabled.ll (84%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-invalid.ll (86%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-livein.ll (91%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-memory-contiguous.ll (95%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-memory-gather-scatter.ll (95%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-memory-interleaved.ll (93%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-predication.ll (93%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-select.ll (97%)
rename llvm/test/Transforms/LoopVectorize/{AArch64 => }/revec-unroll.ll (91%)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-disabled.ll b/llvm/test/Transforms/LoopVectorize/revec-disabled.ll
similarity index 84%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-disabled.ll
rename to llvm/test/Transforms/LoopVectorize/revec-disabled.ll
index ee50a9fca7b61..8b13de534516d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-disabled.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-disabled.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -force-vector-interleave=1 -vectorize-vector-loops=false \
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops=false \
+; RUN: -force-vector-interleave=1 \
; RUN: < %s | FileCheck %s --check-prefix=NO-REVEC
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -force-vector-interleave=2 -vectorize-vector-loops=false \
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops=false \
+; RUN: -force-vector-interleave=2 \
; RUN: < %s | FileCheck %s --check-prefix=NO-REVEC
; Make sure vector loops aren't modified when -vectorize-vector-loops is false.
define void @copy_v4(ptr noalias %dst, ptr noalias %src, i64 %n) {
; NO-REVEC-LABEL: define void @copy_v4(
-; NO-REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
; NO-REVEC-NEXT: [[ENTRY:.*]]:
; NO-REVEC-NEXT: br label %[[LOOP:.*]]
; NO-REVEC: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll b/llvm/test/Transforms/LoopVectorize/revec-invalid.ll
similarity index 86%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
rename to llvm/test/Transforms/LoopVectorize/revec-invalid.ll
index 8695676c6e49d..8102716de3ea4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-invalid.ll
@@ -1,9 +1,9 @@
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s --check-prefix=IR
-; RUN: opt -disable-output -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
-; RUN: -vectorize-vector-loops -pass-remarks-analysis=loop-vectorize \
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: < %s | FileCheck %s --check-prefix=IR
+; RUN: opt -disable-output -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: -pass-remarks-analysis=loop-vectorize \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=REMARKS
; IR-LABEL: @insertelement_cost(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-livein.ll b/llvm/test/Transforms/LoopVectorize/revec-livein.ll
similarity index 91%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-livein.ll
rename to llvm/test/Transforms/LoopVectorize/revec-livein.ll
index 20d284bb10579..0596219702c55 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-livein.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-livein.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: < %s | FileCheck %s
define void @select_livein_v4(ptr noalias %dst, ptr noalias %src, <4 x i32> %threshold, i64 %n) {
; CHECK-LABEL: define void @select_livein_v4(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], <4 x i32> [[THRESHOLD:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], <4 x i32> [[THRESHOLD:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-contiguous.ll b/llvm/test/Transforms/LoopVectorize/revec-memory-contiguous.ll
similarity index 95%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-contiguous.ll
rename to llvm/test/Transforms/LoopVectorize/revec-memory-contiguous.ll
index 128443f63c079..a4e12e251ed43 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-contiguous.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-memory-contiguous.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
; RUN: < %s | FileCheck %s --check-prefix=REVEC
define void @copy_v4(ptr noalias %dst, ptr noalias %src, i64 %n) {
; REVEC-LABEL: define void @copy_v4(
-; REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
; REVEC-NEXT: [[ENTRY:.*:]]
; REVEC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; REVEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
@@ -47,7 +47,7 @@ exit:
define void @copy_v4_nontemporal(ptr noalias %dst, ptr noalias %src, i64 %n) {
; REVEC-LABEL: define void @copy_v4_nontemporal(
-; REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; REVEC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
; REVEC-NEXT: [[ENTRY:.*:]]
; REVEC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; REVEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/revec-memory-gather-scatter.ll
similarity index 95%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-gather-scatter.ll
rename to llvm/test/Transforms/LoopVectorize/revec-memory-gather-scatter.ll
index cbb5aa7d8bd0a..51ff62ece278e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-memory-gather-scatter.ll
@@ -1,11 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: -force-target-supports-gather-scatter-ops \
+; RUN: < %s | FileCheck %s
define void @strided_v4(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK-LABEL: define void @strided_v4(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -41,7 +42,7 @@ exit:
define void @gather_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
; CHECK-LABEL: define void @gather_128(
-; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) {
; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP:.*]]:
@@ -81,7 +82,7 @@ for.body: ; preds = %entry, %for.body
define void @scatter_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
; CHECK-LABEL: define void @scatter_128(
-; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) {
; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP:.*]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-interleaved.ll b/llvm/test/Transforms/LoopVectorize/revec-memory-interleaved.ll
similarity index 93%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-interleaved.ll
rename to llvm/test/Transforms/LoopVectorize/revec-memory-interleaved.ll
index a78af8d6fe473..9a613f50c30be 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-memory-interleaved.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-memory-interleaved.ll
@@ -1,12 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: -enable-interleaved-mem-accesses \
+; RUN: < %s | FileCheck %s
define void @ld2q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
; CHECK-LABEL: define void @ld2q(
-; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP:.*]]:
@@ -46,7 +47,7 @@ for.body: ; preds = %entry, %for.body
define void @st2q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
; CHECK-LABEL: define void @st2q(
-; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP:.*]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll b/llvm/test/Transforms/LoopVectorize/revec-predication.ll
similarity index 93%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
rename to llvm/test/Transforms/LoopVectorize/revec-predication.ll
index f3a3ba7f00541..59de0f29911c9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-predication.ll
@@ -1,11 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: -force-target-supports-masked-memory-ops \
+; RUN: < %s | FileCheck %s
define void @predicated_v4(ptr noalias %dst, ptr noalias %src, i64 %n, i64 %limit) {
; CHECK-LABEL: define void @predicated_v4(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]], i64 [[LIMIT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]], i64 [[LIMIT:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -51,7 +52,7 @@ exit:
define void @predicated_load_uniform_cond(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, i1 %cond) {
; CHECK-LABEL: define void @predicated_load_uniform_cond(
-; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], i1 [[COND:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP:.*]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll b/llvm/test/Transforms/LoopVectorize/revec-select.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
rename to llvm/test/Transforms/LoopVectorize/revec-select.ll
index 2d837a5236e51..a054e053ed033 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-select.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -scalable-vectorization=on -force-vector-interleave=1 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: < %s | FileCheck %s
define void @icmp_select_v4(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
; CHECK-LABEL: define void @icmp_select_v4(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
@@ -55,7 +55,7 @@ exit:
define void @icmp_select_v4_uniform_cond(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n, <4 x i32> %conds) {
; CHECK-LABEL: define void @icmp_select_v4_uniform_cond(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]], <4 x i32> [[CONDS:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]], <4 x i32> [[CONDS:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <4 x i32> [[CONDS]], zeroinitializer
; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v4i1(<4 x i1> [[CMP]])
@@ -106,7 +106,7 @@ exit:
define void @icmp_select_scalar_cond_uniform(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n, i1 %cond) {
; CHECK-LABEL: define void @icmp_select_scalar_cond_uniform(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]], i1 [[COND:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
@@ -155,7 +155,7 @@ exit:
; TODO-REVEC: LV does not handle stretching a nxv1i1 predicate to nxv4i1
define void @icmp_select_scalar_cond_load(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, ptr noalias %cond, i64 %n) {
; CHECK-LABEL: define void @icmp_select_scalar_cond_load(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -202,7 +202,7 @@ exit:
; TODO-REVEC: LV does not handle stretching a nxv1i1 predicate to nxv4i1
define void @icmp_select_scalar_cond_iv(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
; CHECK-LABEL: define void @icmp_select_scalar_cond_iv(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -244,7 +244,7 @@ exit:
define void @fcmp_select_v4(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
; CHECK-LABEL: define void @fcmp_select_v4(
-; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-unroll.ll b/llvm/test/Transforms/LoopVectorize/revec-unroll.ll
similarity index 91%
rename from llvm/test/Transforms/LoopVectorize/AArch64/revec-unroll.ll
rename to llvm/test/Transforms/LoopVectorize/revec-unroll.ll
index feaef7eaaf1d3..2821572e7f797 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/revec-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-unroll.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -force-vector-width=1 -force-vector-interleave=2 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s --check-prefix=VF1IC2
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve \
-; RUN: -force-vector-width="vscale x 1" -force-vector-interleave=2 \
-; RUN: -vectorize-vector-loops < %s | FileCheck %s --check-prefix=SVF1IC2
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=2 -force-vector-width=1 \
+; RUN: < %s | FileCheck %s --check-prefix=VF1IC2
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=2 -force-vector-width="vscale x 1" \
+; RUN: < %s | FileCheck %s --check-prefix=SVF1IC2
; Verify that we can interleave vector loop iterations.
define void @load_add_store(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs, i64 %n) {
; VF1IC2-LABEL: define void @load_add_store(
-; VF1IC2-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; VF1IC2-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) {
; VF1IC2-NEXT: [[ENTRY:.*:]]
; VF1IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; VF1IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -44,7 +44,7 @@ define void @load_add_store(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs
; VF1IC2: [[SCALAR_PH]]:
;
; SVF1IC2-LABEL: define void @load_add_store(
-; SVF1IC2-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; SVF1IC2-SAME: ptr noalias [[DST:%.*]], ptr noalias [[LHS:%.*]], ptr noalias [[RHS:%.*]], i64 [[N:%.*]]) {
; SVF1IC2-NEXT: [[ENTRY:.*:]]
; SVF1IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; SVF1IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
>From 618458604b3a14fb836d3341fae080a100c94849 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 4 Aug 2026 14:04:25 +0000
Subject: [PATCH 3/7] Stop using isElementTypeLegalForScalableVector as type
check for REVEC
---
.../Target/AArch64/AArch64TargetTransformInfo.h | 9 ---------
.../Vectorize/LoopVectorizationLegality.cpp | 15 +++++----------
.../Vectorize/LoopVectorizationPlanner.cpp | 9 ++-------
3 files changed, 7 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 1fb39de0c3ee7..ddca341bad05b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -305,15 +305,6 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
Ty->isIntegerTy(32) || Ty->isIntegerTy(64))
return true;
- // REVEC: Allow 64-bit and 128-bit vectors, as well as i1 vectors. We still
- // avoid creating nxv1 types though, as codegen for those can be suboptimal.
- if (auto *FVTy = dyn_cast<FixedVectorType>(Ty))
- return (FVTy->getElementType()->isIntegerTy(1) ||
- DL.getTypeSizeInBits(FVTy) == AArch64::SVEBitsPerBlock ||
- DL.getTypeSizeInBits(FVTy) == AArch64::SVEBitsPerBlock / 2) &&
- FVTy->getNumElements() > 1 &&
- isElementTypeLegalForScalableVector(FVTy->getElementType());
-
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 2e94484bb3dca..88b72fe4728c5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -975,13 +975,11 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
[](const Value *V) { return V->getType()->isVectorTy(); }))
LoopContainsVectors = true;
- auto CanWidenInstructionTy = [TTI = TTI](Instruction const &Inst) {
+ auto CanWidenInstructionTy = [](Instruction const &Inst) {
Type *InstTy = Inst.getType();
- // TODO-REVEC: To support fixed VFs, we'll need to query a diffent TTI hook.
if (isa<FixedVectorType>(InstTy))
- return VectorizeVectorLoops &&
- TTI->isElementTypeLegalForScalableVector(InstTy->getScalarType());
+ return VectorizeVectorLoops.getValue();
if (!isa<StructType>(InstTy))
return canVectorizeTy(InstTy);
@@ -999,8 +997,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
"CanWidenInstructionTy was not checked beforehand.");
Type *FromTy = CastI.getOperand(0)->getType();
return VectorType::isValidElementType(FromTy) ||
- (isa<FixedVectorType>(FromTy) && VectorizeVectorLoops &&
- TTI->isElementTypeLegalForScalableVector(FromTy->getScalarType()));
+ (isa<FixedVectorType>(FromTy) && VectorizeVectorLoops);
};
// Check that the instruction return type is vectorizable.
@@ -1017,10 +1014,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// Check that the stored type is vectorizable.
if (auto *ST = dyn_cast<StoreInst>(&I)) {
Type *T = ST->getValueOperand()->getType();
- bool CanWidenStoreType =
- VectorType::isValidElementType(T) ||
- (isa<FixedVectorType>(T) && VectorizeVectorLoops &&
- TTI->isElementTypeLegalForScalableVector(T->getScalarType()));
+ bool CanWidenStoreType = VectorType::isValidElementType(T) ||
+ (isa<FixedVectorType>(T) && VectorizeVectorLoops);
if (!CanWidenStoreType) {
reportVectorizationFailure("Store instruction cannot be vectorized",
"CantVectorizeStore", ORE, TheLoop, ST);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index 31b0f879a6925..d1496d71e505a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -351,13 +351,8 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
// Disable scalable vectorization if the loop contains any instructions
// with element types not supported for scalable vectors.
if (any_of(ElementTypesInLoop, [&](Type *Ty) {
- if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
- if (!VectorizeVectorLoops)
- return true;
- Ty = FVTy->getElementType();
- } else if (Ty->isVectorTy())
- return true;
- return !Ty->isVoidTy() && !TTI.isElementTypeLegalForScalableVector(Ty);
+ return !Ty->isVoidTy() &&
+ !TTI.isElementTypeLegalForScalableVector(Ty->getScalarType());
})) {
reportVectorizationInfo("Scalable vectorization is not supported "
"for all element types found in this loop.",
>From 61f1220d19cf8dc2ca4a40da9248dfbd43c979e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 4 Aug 2026 15:31:21 +0000
Subject: [PATCH 4/7] Revert REVEC-specific useMaxBandwidth() change and add
tests
---
.../Vectorize/LoopVectorizationPlanner.cpp | 4 +-
.../AArch64/revec-maximize-bandwidth.ll | 50 +++++++++++++++++++
.../LoopVectorize/revec-maximize-bandwidth.ll | 45 +++++++++++++++++
3 files changed, 96 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-maximize-bandwidth.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/revec-maximize-bandwidth.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index d1496d71e505a..7485b7087d4a0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -278,9 +278,7 @@ ElementCount VFSelectionContext::getMaximizedVFForTarget(
else
MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
- // REVEC: Avoid creating wider than expected scalable types by choosing
- // VF > vscale x 1.
- if (useMaxBandwidth(ComputeScalableMaxVF) && !Legal->LoopContainsVectors) {
+ if (useMaxBandwidth(ComputeScalableMaxVF)) {
auto MaxVectorElementCountMaxBW = ElementCount::get(
llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
ComputeScalableMaxVF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-maximize-bandwidth.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-maximize-bandwidth.ll
new file mode 100644
index 0000000000000..5b05810bb2950
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-maximize-bandwidth.ll
@@ -0,0 +1,50 @@
+; REQUIRES: asserts
+; RUN: opt -disable-output -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve \
+; RUN: -vectorizer-maximize-bandwidth=false -debug-only=loop-vectorize \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=NOMAXBW
+; RUN: opt -disable-output -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve \
+; RUN: -vectorizer-maximize-bandwidth=true -debug-only=loop-vectorize \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=MAXBW
+
+; The widest value fills a 128-bit NEON register. Without maximizing bandwidth,
+; it limits the scalable VF to vscale x 1. The smaller 64-bit value allows
+; vscale x 2 when maximizing bandwidth.
+
+; NOMAXBW-LABEL: LV: Checking a loop in 'max_bandwidth'
+; NOMAXBW: LV: The Smallest and Widest types: 64 / 128 bits.
+; NOMAXBW: LV: The Widest register safe to use is: 128 bits.
+; NOMAXBW-NEXT: LV: The Widest register safe to use is: vscale x 128 bits.
+; NOMAXBW-NEXT: LV: Found feasible scalable VF = vscale x 1
+
+; MAXBW-LABEL: LV: Checking a loop in 'max_bandwidth'
+; MAXBW: LV: The Smallest and Widest types: 64 / 128 bits.
+; MAXBW: LV: The Widest register safe to use is: 128 bits.
+; MAXBW-NEXT: LV: The Widest register safe to use is: vscale x 128 bits.
+; MAXBW-NEXT: LV: Found feasible scalable VF = vscale x 2
+
+define void @max_bandwidth(ptr noalias %dst16, ptr noalias %src16,
+ ptr noalias %dst8, ptr noalias %src8, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src16.gep = getelementptr inbounds <16 x i8>, ptr %src16, i64 %iv
+ %dst16.gep = getelementptr inbounds <16 x i8>, ptr %dst16, i64 %iv
+ %v16 = load <16 x i8>, ptr %src16.gep, align 16
+ %add16 = add <16 x i8> %v16, splat (i8 1)
+ store <16 x i8> %add16, ptr %dst16.gep, align 16
+ %src8.gep = getelementptr inbounds <8 x i8>, ptr %src8, i64 %iv
+ %dst8.gep = getelementptr inbounds <8 x i8>, ptr %dst8, i64 %iv
+ %v8 = load <8 x i8>, ptr %src8.gep, align 8
+ %add8 = add <8 x i8> %v8, splat (i8 1)
+ store <8 x i8> %add8, ptr %dst8.gep, align 8
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/revec-maximize-bandwidth.ll b/llvm/test/Transforms/LoopVectorize/revec-maximize-bandwidth.ll
new file mode 100644
index 0000000000000..adb0698294136
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/revec-maximize-bandwidth.ll
@@ -0,0 +1,45 @@
+; REQUIRES: asserts
+; RUN: opt -disable-output -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-target-supports-scalable-vectors -scalable-vectorization=on \
+; RUN: -vectorizer-maximize-bandwidth=false -debug-only=loop-vectorize \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=NOMAXBW
+; RUN: opt -disable-output -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-target-supports-scalable-vectors -scalable-vectorization=on \
+; RUN: -vectorizer-maximize-bandwidth=true -debug-only=loop-vectorize \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=MAXBW
+
+; Without maximizing bandwidth, the widest type limits the scalable VF to
+; vscale x 1. When maximizing bandwidth, the smallest type allows vscale x 2.
+
+; NOMAXBW-LABEL: LV: Checking a loop in 'max_bandwidth'
+; NOMAXBW: LV: The Smallest and Widest types: 16 / 32 bits.
+; NOMAXBW: LV: Found feasible scalable VF = vscale x 1
+
+; MAXBW-LABEL: LV: Checking a loop in 'max_bandwidth'
+; MAXBW: LV: The Smallest and Widest types: 16 / 32 bits.
+; MAXBW: LV: Found feasible scalable VF = vscale x 2
+
+define void @max_bandwidth(ptr noalias %dst4, ptr noalias %src4,
+ ptr noalias %dst2, ptr noalias %src2, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src4.gep = getelementptr inbounds <4 x i8>, ptr %src4, i64 %iv
+ %dst4.gep = getelementptr inbounds <4 x i8>, ptr %dst4, i64 %iv
+ %v4 = load <4 x i8>, ptr %src4.gep, align 4
+ %add4 = add <4 x i8> %v4, splat (i8 1)
+ store <4 x i8> %add4, ptr %dst4.gep, align 4
+ %src2.gep = getelementptr inbounds <2 x i8>, ptr %src2, i64 %iv
+ %dst2.gep = getelementptr inbounds <2 x i8>, ptr %dst2, i64 %iv
+ %v2 = load <2 x i8>, ptr %src2.gep, align 2
+ %add2 = add <2 x i8> %v2, splat (i8 1)
+ store <2 x i8> %add2, ptr %dst2.gep, align 2
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
>From c7eb31ee1d1c4dcd216b4b8fb591728b20ecbcb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 4 Aug 2026 16:06:43 +0000
Subject: [PATCH 5/7] Restore type checks in computeScalarTypeForInstruction()
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 82a97ac1883f4..60aa9f0862111 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -494,7 +494,12 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
case Instruction::Store:
return Type::getVoidTy(Ctx);
case Instruction::ICmp:
+ assert(Op0Ty->getScalarType()->isIntOrPtrTy() &&
+ "expected integer or pointer operand");
+ AssertOperandType(1, Op0Ty);
+ return CmpInst::makeCmpResultType(Op0Ty);
case Instruction::FCmp:
+ assert(Op0Ty->isFPOrFPVectorTy() && "expected floating-point operand");
AssertOperandType(1, Op0Ty);
return CmpInst::makeCmpResultType(Op0Ty);
case VPInstruction::ActiveLaneMask:
@@ -518,6 +523,8 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
assert(Op0Ty->isIntegerTy() && "expected integer operand");
return IntegerType::get(Ctx, 32);
case Instruction::Select: {
+ assert((!Op0Ty || Op0Ty->isIntOrIntVectorTy(1)) &&
+ "select condition must be bool");
Type *Op1Ty = Operands[1]->getScalarType();
AssertOperandType(2, Op1Ty);
return Op1Ty;
>From 629ecec20c65cea9523f7a3238c53e17e3706bcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 5 Aug 2026 10:00:25 +0000
Subject: [PATCH 6/7] Test get(VPV, LaneIdx) for "vector lanes"
This requires a slight adjustment to type asserts in
VPInstruction::execute to be safe for REVEC.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 ++-
.../Transforms/LoopVectorize/revec-liveout.ll | 77 +++++++++++++++++++
3 files changed, 87 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/revec-liveout.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..cc76225aabd31 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1511,8 +1511,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns true if the recipe only uses the first part of operand \p Op.
bool usesFirstPartOnly(const VPValue *Op) const override;
- /// Returns true if this VPInstruction produces a scalar value from a vector,
+ /// Whether this VPInstruction turns a VFxInitialTy type back into InitialTy,
/// e.g. by performing a reduction or extracting a lane.
+ /// Note InitialTy might already be a vector if re-vectorising.
bool isVectorToScalar() const;
/// Returns true if the recipe produces a single scalar value.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 60aa9f0862111..c50b3f5062297 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1590,10 +1590,14 @@ void VPInstruction::execute(VPTransformState &State) {
bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
(vputils::onlyFirstLaneUsed(this) ||
isVectorToScalar() || isSingleScalar());
- assert((((GeneratedValue->getType()->isVectorTy() ||
- GeneratedValue->getType()->isStructTy()) ==
- !GeneratesPerFirstLaneOnly) ||
- State.VF.isScalar()) &&
+ // REVEC: our initial values might be of vector type.
+ Type *InitialTy = getScalarType();
+ ElementCount NewEC = getElementCount(GeneratedValue->getType());
+ ElementCount InitialEC = getElementCount(InitialTy);
+ bool GotWidened = ElementCount::isKnownGT(NewEC, InitialEC) ||
+ (NewEC.isScalable() && !InitialEC.isScalable()) ||
+ GeneratedValue->getType()->isStructTy();
+ assert(((GotWidened == !GeneratesPerFirstLaneOnly) || State.VF.isScalar()) &&
"scalar value but not only first lane defined");
State.set(this, GeneratedValue,
/*IsScalar*/ GeneratesPerFirstLaneOnly);
diff --git a/llvm/test/Transforms/LoopVectorize/revec-liveout.ll b/llvm/test/Transforms/LoopVectorize/revec-liveout.ll
new file mode 100644
index 0000000000000..21e66168fadb0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/revec-liveout.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 1" \
+; RUN: < %s | FileCheck %s --check-prefixes=SVF1
+; RUN: opt -S -passes=loop-vectorize -vectorize-vector-loops \
+; RUN: -force-vector-interleave=1 -force-vector-width="vscale x 2" \
+; RUN: < %s | FileCheck %s --check-prefixes=SVF2
+
+; Check that a vector value live-out is correctly extracted using vector.splice
+; and vector.extract. For extracting the last <4 x i32> SegmentTy in a re-vectorised
+; value, the latter is "spliced right" by the element count of SegmentTy, i.e. 4.
+define <4 x i32> @vector_live_out(ptr noalias %src, i64 %n) {
+; SVF1-LABEL: define <4 x i32> @vector_live_out(
+; SVF1-SAME: ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; SVF1-NEXT: [[ENTRY:.*:]]
+; SVF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; SVF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0]]
+; SVF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SVF1: [[VECTOR_PH]]:
+; SVF1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; SVF1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; SVF1-NEXT: br label %[[VECTOR_BODY:.*]]
+; SVF1: [[VECTOR_BODY]]:
+; SVF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SVF1-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[INDEX]]
+; SVF1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 16
+; SVF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; SVF1-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SVF1-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; SVF1: [[MIDDLE_BLOCK]]:
+; SVF1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> poison, i32 4)
+; SVF1-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP3]], i64 0)
+; SVF1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; SVF1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; SVF1: [[SCALAR_PH]]:
+;
+; SVF2-LABEL: define <4 x i32> @vector_live_out(
+; SVF2-SAME: ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; SVF2-NEXT: [[ENTRY:.*:]]
+; SVF2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; SVF2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; SVF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; SVF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SVF2: [[VECTOR_PH]]:
+; SVF2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; SVF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; SVF2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; SVF2-NEXT: br label %[[VECTOR_BODY:.*]]
+; SVF2: [[VECTOR_BODY]]:
+; SVF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SVF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[INDEX]]
+; SVF2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i32>, ptr [[TMP3]], align 16
+; SVF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; SVF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SVF2-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; SVF2: [[MIDDLE_BLOCK]]:
+; SVF2-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i32> @llvm.vector.splice.right.nxv8i32(<vscale x 8 x i32> [[WIDE_LOAD]], <vscale x 8 x i32> poison, i32 4)
+; SVF2-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv8i32(<vscale x 8 x i32> [[TMP5]], i64 0)
+; SVF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; SVF2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; SVF2: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds <4 x i32>, ptr %src, i64 %iv
+ %v = load <4 x i32>, ptr %p, align 16
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ %result = phi <4 x i32> [ %v, %loop ]
+ ret <4 x i32> %result
+}
>From 31ba2c400a266dd6aa898916b397deed42083cdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 7 Aug 2026 10:52:08 +0000
Subject: [PATCH 7/7] Update test after rebase
---
llvm/test/Transforms/LoopVectorize/revec-unroll.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/revec-unroll.ll b/llvm/test/Transforms/LoopVectorize/revec-unroll.ll
index 2821572e7f797..d977c2963f989 100644
--- a/llvm/test/Transforms/LoopVectorize/revec-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/revec-unroll.ll
@@ -15,7 +15,7 @@ define void @load_add_store(ptr noalias %dst, ptr noalias %lhs, ptr noalias %rhs
; VF1IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; VF1IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF1IC2: [[VECTOR_PH]]:
-; VF1IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; VF1IC2-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; VF1IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; VF1IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF1IC2: [[VECTOR_BODY]]:
More information about the llvm-branch-commits
mailing list