[llvm] [LV] Add support for widening loads/stores to a UF x VF (PR #217670)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 02:57:30 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/217670
>From 759c1bb40a2857b21c475421e047da6b6ed5c2bf Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 20 Aug 2026 14:39:53 +0000
Subject: [PATCH 01/21] [LV] Add support for widening loads/stores to a VF
multiple
This patch adds support for widening loads and stores by a "VF multiple"
that must divide the UF. It is currently limited to unmasked operations,
but we plan to extend it to masked loops via the wide active lane mask.
For now, this is driven by a new TTI hook,
`getPreferredVFMultipleForMemoryOp`. A small VPlan transform uses that
hook to set the VF multiple on `VPWidenLoadRecipe` and
`VPWidenStoreRecipe`.
When VPlan unrolling encounters a load or store with a VF multiple
greater than 1, it inserts extracts for the unroll parts of widened
loads and concatenates multiple unroll parts for widened stores.
On AArch64 this is used to target multi-vector load/store instructions.
---
.../llvm/Analysis/TargetTransformInfo.h | 9 +
.../llvm/Analysis/TargetTransformInfoImpl.h | 8 +
llvm/lib/Analysis/TargetTransformInfo.cpp | 7 +
.../AArch64/AArch64TargetTransformInfo.cpp | 31 ++
.../AArch64/AArch64TargetTransformInfo.h | 4 +
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +
llvm/lib/Transforms/Vectorize/VPlan.h | 27 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 37 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 28 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 41 ++
.../Transforms/Vectorize/VPlanTransforms.h | 5 +
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 66 ++++
.../AArch64/multi-vector-mem-ops.ll | 374 ++++++++++++++++++
.../vplan-printing-multi-vector-mem-ops.ll | 151 +++++++
.../VPlan/vplan-print-before-after-all.ll | 1 +
15 files changed, 784 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index e81809cca64f8..6795267dad097 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1001,6 +1001,15 @@ class TargetTransformInfo {
unsigned Opcode1,
const SmallBitVector &OpcodeMask) const;
+ /// Return the preferred multiple of VF to use for a contiguous load/store.
+ /// Returning 1 leaves the operation at VF. The returned value must divide UF.
+ ///
+ /// \p Opcode must be either Instruction::Load or Instruction::Store.
+ LLVM_ABI unsigned
+ getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataType,
+ ElementCount VF, unsigned UF,
+ bool IsMasked = false) const;
+
/// Return true if we should be enabling ordered reductions for the target.
LLVM_ABI bool enableOrderedReductions() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index a19c122c16f20..01006cfce4d08 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -429,6 +429,14 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
+ virtual unsigned getPreferredVFMultipleForMemoryOp(unsigned Opcode,
+ Type *DataType,
+ ElementCount VF,
+ unsigned UF,
+ bool IsMasked) const {
+ return 1;
+ }
+
virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
Align Alignment,
unsigned AddrSpace) const {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 1055bb59148b3..05cd06e173200 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -554,6 +554,13 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
+unsigned TargetTransformInfo::getPreferredVFMultipleForMemoryOp(
+ unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
+ bool IsMasked) const {
+ return TTIImpl->getPreferredVFMultipleForMemoryOp(Opcode, DataType, VF, UF,
+ IsMasked);
+}
+
bool TargetTransformInfo::isLegalInterleavedAccessType(
VectorType *VTy, unsigned Factor, Align Alignment,
unsigned AddrSpace) const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 1c815c439aca0..9fa53493c7d20 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5974,6 +5974,37 @@ bool AArch64TTIImpl::isLegalSpeculativeLoad(Type *DataType,
Size.getFixedValue() <= 16;
}
+unsigned
+AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataTy,
+ ElementCount VF, unsigned UF,
+ bool IsMasked) const {
+ assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
+ "expected load/store opcode");
+ if (IsMasked)
+ return 1; // TODO: Support masked multi-vector loads/stores.
+
+ if (!ST->enableSubRegLiveness())
+ return 1;
+
+ if ((Opcode != Instruction::Load && Opcode != Instruction::Store) ||
+ !ST->hasSVE2p1() || !VF.isScalable() || !isPowerOf2_32(UF))
+ return 1;
+
+ unsigned VectorWidth = VF.getKnownMinValue() * DL.getTypeSizeInBits(DataTy);
+ if (VectorWidth % 128 != 0)
+ return 1;
+
+ for (unsigned TargetWidth : {512u, 256u}) {
+ if (TargetWidth % VectorWidth == 0) {
+ unsigned Scale = TargetWidth / VectorWidth;
+ if (Scale <= UF)
+ return Scale;
+ }
+ }
+
+ return 1;
+}
+
unsigned
AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF,
bool HasUnorderedReductions) const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index d090f69c1476a..a5513114bd112 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -282,6 +282,10 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isLegalSpeculativeLoad(Type *DataType,
unsigned AddressSpace) const override;
+ unsigned getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataType,
+ ElementCount VF, unsigned UF,
+ bool IsMasked) const override;
+
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE) const override;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d28760ae36397..c0a8007e25192 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5792,6 +5792,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
*PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
+ RUN_VPLAN_PASS(VPlanTransforms::scaleMemoryAccessesByUF, BestVPlan, BestVF,
+ BestUF, CM.TTI);
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b1b6b6dd937b8..0edf821a0806f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1319,6 +1319,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
WideActiveLaneMask,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
+ // Concatenates its unrolled part operands into one widened vector.
+ ConcatVectorParts,
ExplicitVectorLength,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
@@ -3775,6 +3777,10 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
/// Whether the memory access is masked.
bool IsMasked = false;
+ /// Multiple of VF used to widen this memory operation. The final operation
+ /// loads or stores VF * VFMultiple elements
+ unsigned VFMultiple = 1;
+
void setMask(VPValue *Mask) {
assert(!IsMasked && "cannot re-set mask");
if (!Mask)
@@ -3821,6 +3827,12 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
Instruction &getIngredient() const { return Ingredient; }
+
+ /// Set the VF multiple for this memory operation.
+ void setVFMultiple(unsigned VFMultiple) { this->VFMultiple = VFMultiple; }
+
+ /// Returns the VF multiple of this memory operation.
+ unsigned getVFMultiple() const { return VFMultiple; }
};
/// A recipe for widening load operations, using the address to load from and an
@@ -3836,8 +3848,11 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPSingleDefRecipe,
}
VPWidenLoadRecipe *clone() override {
- return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
- getMask(), Consecutive, *this, getDebugLoc());
+ auto *R =
+ new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(), getMask(),
+ Consecutive, *this, getDebugLoc());
+ R->setVFMultiple(VFMultiple);
+ return R;
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadSC);
@@ -3941,9 +3956,11 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPRecipeBase,
}
VPWidenStoreRecipe *clone() override {
- return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
- getStoredValue(), getMask(), Consecutive,
- *this, getDebugLoc());
+ auto *R = new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
+ getStoredValue(), getMask(), Consecutive,
+ *this, getDebugLoc());
+ R->setVFMultiple(VFMultiple);
+ return R;
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStoreSC);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index d795f9748f1ba..22f35f1b96ecd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -292,7 +292,10 @@ struct Recipe_match {
// Check for recipes that do not have opcodes.
if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
- std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>)
+ std::is_same_v<RecipeTy, VPVectorEndPointerRecipe> ||
+ std::is_same_v<RecipeTy, VPVectorPointerRecipe> ||
+ std::is_same_v<RecipeTy, VPWidenLoadRecipe> ||
+ std::is_same_v<RecipeTy, VPWidenStoreRecipe>)
return DefR;
else
return DefR && DefR->getOpcode() == Opcode;
@@ -1049,6 +1052,38 @@ m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
}
+template <typename Op0_t, typename Op1_t>
+using VectorPointerRecipe_match =
+ Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
+ /*Commutative*/ false, VPVectorPointerRecipe>;
+
+template <typename Op0_t, typename Op1_t>
+VectorPointerRecipe_match<Op0_t, Op1_t> m_VecPtr(const Op0_t &Op0,
+ const Op1_t &Op1) {
+ return VectorPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
+}
+
+template <typename Op0_t>
+using VPWidenLoadRecipe_match =
+ Recipe_match<std::tuple<Op0_t>, 0,
+ /*Commutative*/ false, VPWidenLoadRecipe>;
+
+template <typename Op0_t>
+VPWidenLoadRecipe_match<Op0_t> m_WidenLoad(const Op0_t &Op0) {
+ return VPWidenLoadRecipe_match<Op0_t>(Op0);
+}
+
+template <typename Op0_t, typename Op1_t>
+using VPWidenStoreRecipe_match =
+ Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
+ /*Commutative*/ false, VPWidenStoreRecipe>;
+
+template <typename Op0_t, typename Op1_t>
+VPWidenStoreRecipe_match<Op0_t, Op1_t> m_WidenStore(const Op0_t &Op0,
+ const Op1_t &Op1) {
+ return VPWidenStoreRecipe_match<Op0_t, Op1_t>(Op0, Op1);
+}
+
template <typename Op0_t, typename Op1_t>
using VectorEndPointerRecipe_match =
Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ed18486f538d4..6af97a147f239 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -564,6 +564,9 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
return StructTy->getTypeAtIndex(
cast<VPConstantInt>(Operands[1])->getZExtValue());
}
+ case VPInstruction::ConcatVectorParts:
+ case VPInstruction::ExtractVectorForPart:
+ return Op0Ty;
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::NumActiveLanes:
@@ -691,6 +694,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLane:
case VPInstruction::ExtractLastActive:
+ case VPInstruction::ConcatVectorParts:
// Cannot determine the number of operands from the opcode.
return -1u;
}
@@ -1180,6 +1184,19 @@ Value *VPInstruction::generate(VPTransformState &State,
vputils::getIntrinsicID(this), Args,
/*FMFSource=*/nullptr, getName());
}
+ case VPInstruction::ConcatVectorParts: {
+ unsigned VectorOps = getNumOperands();
+ auto *WideDataTy = VectorType::get(
+ getScalarType(), State.VF.multiplyCoefficientBy(VectorOps));
+ Value *WideData = PoisonValue::get(WideDataTy);
+
+ for (unsigned I = 0; I < VectorOps; ++I) {
+ Value *Part = State.get(getOperand(I));
+ WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part,
+ I * State.VF.getKnownMinValue());
+ }
+ return WideData;
+ }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -1729,6 +1746,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::IncomingAliasMask:
+ case VPInstruction::ConcatVectorParts:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::FirstActiveLane:
@@ -1865,6 +1883,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
break;
+ case VPInstruction::ConcatVectorParts:
+ O << "concat-vector-parts";
+ break;
case VPInstruction::ExplicitVectorLength:
O << "EXPLICIT-VECTOR-LENGTH";
break;
@@ -4373,7 +4394,8 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
void VPWidenLoadRecipe::execute(VPTransformState &State) {
Type *ScalarDataTy = getScalarType();
- auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
+ auto *DataTy =
+ VectorType::get(ScalarDataTy, State.VF.multiplyCoefficientBy(VFMultiple));
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
@@ -4403,6 +4425,8 @@ void VPWidenLoadRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN ";
printAsOperand(O, SlotTracker);
O << " = load ";
+ if (VFMultiple > 1)
+ O << "x" << VFMultiple << ' ';
printOperands(O, SlotTracker);
}
#endif
@@ -4490,6 +4514,8 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
void VPWidenStoreRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN store ";
+ if (VFMultiple > 1)
+ O << "x" << VFMultiple << ' ';
printOperands(O, SlotTracker);
}
#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bada5e597ae48..52235e7b7ebb8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4196,6 +4196,47 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
}
}
+void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
+ unsigned UF,
+ const TargetTransformInfo &TTI) {
+ if (UF == 1)
+ return;
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ uint64_t Stride;
+ VPValue *StoredValue = nullptr;
+ auto m_ConstantStrideVecPtr =
+ m_VecPtr(m_VPValue(), m_ConstantInt(Stride));
+ if ((!match(&R, m_WidenLoad(m_ConstantStrideVecPtr)) &&
+ !match(&R, m_WidenStore(m_ConstantStrideVecPtr,
+ m_VPValue(StoredValue)))) ||
+ Stride != 1)
+ continue;
+
+ auto *MemOp = cast<VPWidenMemoryRecipe>(&R);
+ if (!MemOp->isConsecutive())
+ continue;
+
+ // TODO: Support masked loads/stores. This requires widening the header
+ // mask to the same factor as the memory operation.
+ assert(!MemOp->isMasked() && "Masked accesses are not supported yet");
+
+ Type *AccessType = StoredValue ? StoredValue->getScalarType()
+ : R.getVPSingleValue()->getScalarType();
+ unsigned Opcode = isa<VPWidenLoadRecipe>(MemOp->getAsRecipe())
+ ? Instruction::Load
+ : Instruction::Store;
+ unsigned ScaleFactor = TTI.getPreferredVFMultipleForMemoryOp(
+ Opcode, AccessType, VF, UF, /*IsMasked=*/false);
+ assert((ScaleFactor != 0 && UF % ScaleFactor == 0) &&
+ "ScaleFactor must divide UF");
+ MemOp->setVFMultiple(ScaleFactor);
+ }
+ }
+}
+
/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index fdf17d07d70b5..f59beb0e6b1e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -482,6 +482,11 @@ struct VPlanTransforms {
static void sinkPredicatedStores(VPlan &Plan, PredicatedScalarEvolution &PSE,
const Loop *L);
+ /// Widens memory operations by a factor of UF based on a target hook.
+ /// This allows targets to use wider memory operations when profitable.
+ static void scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF, unsigned UF,
+ const TargetTransformInfo &TTI);
+
// Materialize vector trip counts for constants early if it can simply be
// computed as (Original TC / VF * UF) * VF * UF.
static void
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 43e3d08291bdc..29b33f607952e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -74,6 +74,9 @@ class UnrollState {
return Plan.getConstantInt(CanIVIntTy, Part);
}
+ /// Unroll a VPWidenLoadRecipe or VPWidenStoreRecipe with a VFMultiple > 1.
+ void unrollMemOpWithVFMultiple(VPRecipeBase &R, unsigned VFMultiple);
+
public:
UnrollState(VPlan &Plan, unsigned UF) : Plan(Plan), UF(UF) {}
@@ -290,6 +293,62 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
}
}
+void UnrollState::unrollMemOpWithVFMultiple(VPRecipeBase &R,
+ unsigned VFMultiple) {
+ assert(VFMultiple > 1 && UF % VFMultiple == 0);
+ SmallVector<VPRecipeBase *, 4> Groups(UF / VFMultiple, nullptr);
+ Groups[0] = &R;
+
+ // A memory op with a VFMultiple is widened to VF * VFMultiple elements, so
+ // after unrolling by UF we materialize UF / VFMultiple such ops, each
+ // covering VFMultiple unroll parts.
+ VPBuilder Builder = VPBuilder::getToInsertAfter(&R);
+ for (unsigned Group = 1; Group < Groups.size(); ++Group) {
+ auto *Copy = Builder.insert(R.clone());
+ remapOperands(Copy, Group * VFMultiple);
+ Groups[Group] = Copy;
+ }
+
+ if (auto *Store = dyn_cast<VPWidenStoreRecipe>(&R)) {
+ VPValue *StoredValue = Store->getStoredValue();
+ for (unsigned Group = 0; Group < Groups.size(); ++Group) {
+ VPRecipeBase *Store = Groups[Group];
+ Builder.setInsertPoint(Store);
+ SmallVector<VPValue *, 4> Parts;
+ // We need to concatenate VFMultiple parts to form the stored value.
+ for (unsigned Part = 0; Part < VFMultiple; ++Part)
+ Parts.push_back(
+ getValueForPart(StoredValue, Group * VFMultiple + Part));
+ auto *Concat =
+ Builder.createNaryOp(VPInstruction::ConcatVectorParts, Parts);
+ Groups[Group]->setOperand(1, Concat);
+ ToSkip.insert(Concat);
+ }
+ } else {
+ assert(isa<VPWidenLoadRecipe>(R) && "Expected a load recipe");
+ // We need to extract each unroll part as a subvector.
+ auto ExtractPart0 = Builder.createNaryOp(
+ VPInstruction::ExtractVectorForPart,
+ {Groups[0]->getVPSingleValue(), getConstantInt(0)});
+ // First replace R with an extract of the first unroll part (ExtractPart0).
+ R.getVPSingleValue()->replaceUsesWithIf(
+ ExtractPart0, [&](VPUser &U, unsigned) { return &U != ExtractPart0; });
+ ToSkip.insert(ExtractPart0);
+
+ // Create extracts for the remaining unroll parts and remap later uses of
+ // ExtractPart0 to the correct unrolled part.
+ for (unsigned Part = 1; Part != UF; ++Part) {
+ VPRecipeBase *Group = Groups[Part / VFMultiple];
+ unsigned IndexInGroup = Part % VFMultiple;
+ auto *Extract = Builder.createNaryOp(
+ VPInstruction::ExtractVectorForPart,
+ {Group->getVPSingleValue(), getConstantInt(IndexInGroup)});
+ addRecipeForPart(ExtractPart0, Extract, Part);
+ ToSkip.insert(Extract);
+ }
+ }
+}
+
/// Handle non-header-phi recipes.
void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
if (match(&R, m_CombineOr(m_BranchOnCond(), m_BranchOnCount())))
@@ -301,6 +360,13 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
return;
}
}
+
+ if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(&R);
+ WidenMem && WidenMem->getVFMultiple() > 1) {
+ unrollMemOpWithVFMultiple(R, WidenMem->getVFMultiple());
+ return;
+ }
+
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
if (isa<StoreInst>(RepR->getUnderlyingValue()) &&
RepR->getOperand(1)->isDefinedOutsideLoopRegions()) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
new file mode 100644
index 0000000000000..f20e7aff70ef7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -0,0 +1,374 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -sve-tail-folding=disabled %s | FileCheck %s --check-prefix=UNMASKED-SVE2P1
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @mixed_i64_i32_accesses(
+; UNMASKED-SVE2P1-LABEL: define void @mixed_i64_i32_accesses(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[TMP3]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i64 [[TMP14]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i64>, ptr [[TMP15]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP17]], ptr [[TMP5]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP13]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP19]], ptr [[TMP15]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i32>, ptr [[TMP20]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = add <vscale x 4 x i32> [[TMP23]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = add <vscale x 4 x i32> [[TMP24]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP25]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP30]], <vscale x 4 x i32> [[TMP27]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP31]], <vscale x 4 x i32> [[TMP28]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP32]], ptr [[TMP20]], align 4
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+ ptr noalias %x, ptr noalias %y, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %px = getelementptr inbounds i64, ptr %x, i64 %iv
+ %vx = load i64, ptr %px, align 8
+ %ax = add i64 %vx, 1
+ store i64 %ax, ptr %px, align 8
+ %py = getelementptr inbounds i32, ptr %y, i64 %iv
+ %vy = load i32, ptr %py, align 4
+ %ay = add i32 %vy, 1
+ store i32 %ay, ptr %py, align 4
+ %next = add nuw i64 %iv, 1
+ %cmp = icmp ult i64 %next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @mixed_i32_more_frequent_than_i64(
+; UNMASKED-SVE2P1-LABEL: define void @mixed_i32_more_frequent_than_i64(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[TMP3]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i64 [[TMP14]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i64>, ptr [[TMP15]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD3]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD3]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP17]], ptr [[TMP5]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP13]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP19]], ptr [[TMP15]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i32>, ptr [[TMP20]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = add <vscale x 4 x i32> [[TMP23]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = add <vscale x 4 x i32> [[TMP24]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP25]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP30]], <vscale x 4 x i32> [[TMP27]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP31]], <vscale x 4 x i32> [[TMP28]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP32]], ptr [[TMP20]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i32>, ptr [[TMP33]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP36:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP37:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], splat (i32 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP39:%.*]] = add <vscale x 4 x i32> [[TMP35]], splat (i32 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP40:%.*]] = add <vscale x 4 x i32> [[TMP36]], splat (i32 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP41:%.*]] = add <vscale x 4 x i32> [[TMP37]], splat (i32 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP42:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP38]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP43:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP42]], <vscale x 4 x i32> [[TMP39]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP44:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP43]], <vscale x 4 x i32> [[TMP40]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP45:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP44]], <vscale x 4 x i32> [[TMP41]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP45]], ptr [[TMP33]], align 4
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP46]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+ ptr noalias %x, ptr noalias %y, ptr noalias %z, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %px = getelementptr inbounds i64, ptr %x, i64 %iv
+ %vx = load i64, ptr %px, align 8
+ %ax = add i64 %vx, 1
+ store i64 %ax, ptr %px, align 8
+ %py = getelementptr inbounds i32, ptr %y, i64 %iv
+ %vy = load i32, ptr %py, align 4
+ %ay = add i32 %vy, 1
+ store i32 %ay, ptr %py, align 4
+ %pz = getelementptr inbounds i32, ptr %z, i64 %iv
+ %vz = load i32, ptr %pz, align 4
+ %az = add i32 %vz, 2
+ store i32 %az, ptr %pz, align 4
+ %next = add nuw i64 %iv, 1
+ %cmp = icmp ult i64 %next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @first_order_recurrence_i64_scaled_load_and_store(
+; UNMASKED-SVE2P1-LABEL: define void @first_order_recurrence_i64_scaled_load_and_store(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[FIRST:%.*]] = load i64, ptr [[SRC]], align 8
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <2 x i64> poison, i64 [[FIRST]], i32 1
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD3:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = add nuw i64 1, [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 2
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 4
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 6
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3]] = load <2 x i64>, ptr [[TMP4]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[VECTOR_RECUR]], <2 x i64> [[WIDE_LOAD]], <2 x i32> <i32 1, i32 2>
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD]], <2 x i64> [[WIDE_LOAD1]], <2 x i32> <i32 1, i32 2>
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[WIDE_LOAD2]], <2 x i32> <i32 1, i32 2>
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD2]], <2 x i64> [[WIDE_LOAD3]], <2 x i32> <i32 1, i32 2>
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = add <2 x i64> [[TMP5]], [[WIDE_LOAD]]
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <2 x i64> [[TMP6]], [[WIDE_LOAD1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <2 x i64> [[TMP7]], [[WIDE_LOAD2]]
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <2 x i64> [[TMP8]], [[WIDE_LOAD3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 2
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 4
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 6
+; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP13]], align 8
+; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP10]], ptr [[TMP14]], align 8
+; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP11]], ptr [[TMP15]], align 8
+; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP12]], ptr [[TMP16]], align 8
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+ ptr noalias %src, ptr noalias %dst) {
+entry:
+ %first = load i64, ptr %src, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %next, %loop ]
+ %prev = phi i64 [ %first, %entry ], [ %cur, %loop ]
+ %cur.ptr = getelementptr inbounds i64, ptr %src, i64 %iv
+ %cur = load i64, ptr %cur.ptr, align 8
+ %sum = add i64 %prev, %cur
+ %dst.ptr = getelementptr inbounds i64, ptr %dst, i64 %iv
+ store i64 %sum, ptr %dst.ptr, align 8
+ %next = add nuw i64 %iv, 1
+ %cmp = icmp eq i64 %next, 1025
+ br i1 %cmp, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
+; UNMASKED-SVE2P1-LABEL: define i64 @i64_sum_reduction_scaled_partial_reduce(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 3
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP4]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
+; UNMASKED-SVE2P1-NEXT: [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP5]]
+; UNMASKED-SVE2P1-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP6]]
+; UNMASKED-SVE2P1-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP7]]
+; UNMASKED-SVE2P1-NEXT: [[TMP12]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP8]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %acc = phi i64 [ 0, %entry ], [ %sum, %loop ]
+ %p = getelementptr inbounds i64, ptr %a, i64 %iv
+ %ld = load i64, ptr %p, align 8
+ %sum = add i64 %acc, %ld
+ %next = add nuw i64 %iv, 1
+ %cmp = icmp ult i64 %next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret i64 %sum
+}
+
+define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold) {
+; UNMASKED-SVE2P1-LABEL: define i64 @find_last_i64_scaled_load(
+; UNMASKED-SVE2P1-SAME: i64 [[N:%.*]], ptr noalias [[DATA:%.*]], i64 [[THRESHOLD:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 3
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[THRESHOLD]], i64 0
+; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP7]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP8]]
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP11]]
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = freeze <vscale x 2 x i1> [[TMP12]]
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = freeze <vscale x 2 x i1> [[TMP13]]
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = or <vscale x 2 x i1> [[TMP16]], [[TMP17]]
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = freeze <vscale x 2 x i1> [[TMP14]]
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = or <vscale x 2 x i1> [[TMP18]], [[TMP19]]
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = freeze <vscale x 2 x i1> [[TMP15]]
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = or <vscale x 2 x i1> [[TMP20]], [[TMP21]]
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP22]])
+; UNMASKED-SVE2P1-NEXT: [[TMP24]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP25]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP13]], <vscale x 2 x i1> [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP26]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> [[TMP5]]
+; UNMASKED-SVE2P1-NEXT: [[TMP27]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP6]]
+; UNMASKED-SVE2P1-NEXT: [[TMP28]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[VEC_PHI]]
+; UNMASKED-SVE2P1-NEXT: [[TMP29]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP30]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI2]]
+; UNMASKED-SVE2P1-NEXT: [[TMP31]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[VEC_PHI3]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %data.phi = phi i64 [ -1, %entry ], [ %select.data, %loop ]
+ %p = getelementptr inbounds i64, ptr %data, i64 %iv
+ %ld = load i64, ptr %p, align 8
+ %select.cmp = icmp slt i64 %threshold, %ld
+ %select.data = select i1 %select.cmp, i64 %ld, i64 %data.phi
+ %next = add nuw i64 %iv, 1
+ %exit.cmp = icmp eq i64 %next, %n
+ br i1 %exit.cmp, label %exit, label %loop
+
+exit:
+ ret i64 %select.data
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
new file mode 100644
index 0000000000000..3943cb5f28103
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-before="scaleMemoryAccessesByUF$" 2>&1 \
+; RUN: | FileCheck --check-prefix=BEFORE-SCALE %s
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="scaleMemoryAccessesByUF$" 2>&1 \
+; RUN: | FileCheck --check-prefix=AFTER-SCALE %s
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="unrollByUF$" 2>&1 \
+; RUN: | FileCheck --check-prefix=AFTER-UNROLL %s
+
+define void @i64_load_store(ptr noalias %x, i64 %n) {
+; BEFORE-SCALE-LABEL: VPlan for loop in 'i64_load_store'
+; BEFORE-SCALE: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
+; BEFORE-SCALE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; BEFORE-SCALE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; BEFORE-SCALE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; BEFORE-SCALE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; BEFORE-SCALE-EMPTY:
+; BEFORE-SCALE-NEXT: ir-bb<entry>:
+; BEFORE-SCALE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 umax %n)
+; BEFORE-SCALE-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = call i64 @llvm.vscale()
+; BEFORE-SCALE-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = mul nuw vp<[[VP4]]>, ir<16>
+; BEFORE-SCALE-NEXT: EMIT vp<%min.iters.check> = icmp ult vp<[[VP3]]>, vp<[[VP5]]>
+; BEFORE-SCALE-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; BEFORE-SCALE-NEXT: Successor(s): scalar.ph, vector.ph
+; BEFORE-SCALE-EMPTY:
+; BEFORE-SCALE-NEXT: vector.ph:
+; BEFORE-SCALE-NEXT: Successor(s): vector loop
+; BEFORE-SCALE-EMPTY:
+; BEFORE-SCALE-NEXT: <x1> vector loop: {
+; BEFORE-SCALE-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
+; BEFORE-SCALE-EMPTY:
+; BEFORE-SCALE-NEXT: vector.body:
+; BEFORE-SCALE-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
+; BEFORE-SCALE-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
+; BEFORE-SCALE-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; BEFORE-SCALE-NEXT: WIDEN ir<%ld> = load vp<[[VP9]]>
+; BEFORE-SCALE-NEXT: WIDEN ir<%add> = add ir<%ld>, ir<1>
+; BEFORE-SCALE-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; BEFORE-SCALE-NEXT: WIDEN store vp<[[VP10]]>, ir<%add>
+; BEFORE-SCALE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
+; BEFORE-SCALE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; BEFORE-SCALE-NEXT: No successors
+; BEFORE-SCALE-NEXT: }
+; BEFORE-SCALE-NEXT: Successor(s): middle.block
+; BEFORE-SCALE-EMPTY:
+; BEFORE-SCALE-NEXT: middle.block:
+;
+; AFTER-SCALE-LABEL: VPlan for loop in 'i64_load_store'
+; AFTER-SCALE: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
+; AFTER-SCALE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; AFTER-SCALE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; AFTER-SCALE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; AFTER-SCALE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; AFTER-SCALE-EMPTY:
+; AFTER-SCALE-NEXT: ir-bb<entry>:
+; AFTER-SCALE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 umax %n)
+; AFTER-SCALE-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = call i64 @llvm.vscale()
+; AFTER-SCALE-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = mul nuw vp<[[VP4]]>, ir<16>
+; AFTER-SCALE-NEXT: EMIT vp<%min.iters.check> = icmp ult vp<[[VP3]]>, vp<[[VP5]]>
+; AFTER-SCALE-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; AFTER-SCALE-NEXT: Successor(s): scalar.ph, vector.ph
+; AFTER-SCALE-EMPTY:
+; AFTER-SCALE-NEXT: vector.ph:
+; AFTER-SCALE-NEXT: Successor(s): vector loop
+; AFTER-SCALE-EMPTY:
+; AFTER-SCALE-NEXT: <x1> vector loop: {
+; AFTER-SCALE-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
+; AFTER-SCALE-EMPTY:
+; AFTER-SCALE-NEXT: vector.body:
+; AFTER-SCALE-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
+; AFTER-SCALE-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
+; AFTER-SCALE-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-SCALE-NEXT: WIDEN ir<%ld> = load x2 vp<[[VP9]]>
+; AFTER-SCALE-NEXT: WIDEN ir<%add> = add ir<%ld>, ir<1>
+; AFTER-SCALE-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-SCALE-NEXT: WIDEN store x2 vp<[[VP10]]>, ir<%add>
+; AFTER-SCALE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
+; AFTER-SCALE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; AFTER-SCALE-NEXT: No successors
+; AFTER-SCALE-NEXT: }
+; AFTER-SCALE-NEXT: Successor(s): middle.block
+; AFTER-SCALE-EMPTY:
+; AFTER-SCALE-NEXT: middle.block:
+;
+; AFTER-UNROLL-LABEL: VPlan for loop in 'i64_load_store'
+; AFTER-UNROLL: VPlan 'Initial VPlan for VF={vscale x 4},UF={4}' {
+; AFTER-UNROLL-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; AFTER-UNROLL-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; AFTER-UNROLL-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; AFTER-UNROLL-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; AFTER-UNROLL-EMPTY:
+; AFTER-UNROLL-NEXT: ir-bb<entry>:
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 umax %n)
+; AFTER-UNROLL-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = call i64 @llvm.vscale()
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = mul nuw vp<[[VP4]]>, ir<16>
+; AFTER-UNROLL-NEXT: EMIT vp<%min.iters.check> = icmp ult vp<[[VP3]]>, vp<[[VP5]]>
+; AFTER-UNROLL-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; AFTER-UNROLL-NEXT: Successor(s): scalar.ph, vector.ph
+; AFTER-UNROLL-EMPTY:
+; AFTER-UNROLL-NEXT: vector.ph:
+; AFTER-UNROLL-NEXT: Successor(s): vector loop
+; AFTER-UNROLL-EMPTY:
+; AFTER-UNROLL-NEXT: <x1> vector loop: {
+; AFTER-UNROLL-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
+; AFTER-UNROLL-EMPTY:
+; AFTER-UNROLL-NEXT: vector.body:
+; AFTER-UNROLL-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
+; AFTER-UNROLL-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
+; AFTER-UNROLL-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-UNROLL-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP9]]>
+; AFTER-UNROLL-NEXT: WIDEN ir<%ld> = load x2 vp<[[VP10]]>
+; AFTER-UNROLL-NEXT: WIDEN ir<%ld>.1 = load x2 vp<[[VP11]]>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-vector-for-part ir<%ld>, ir<0>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-vector-for-part ir<%ld>, ir<1>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-vector-for-part ir<%ld>.1, ir<0>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-vector-for-part ir<%ld>.1, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add> = add vp<[[VP12]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.1 = add vp<[[VP13]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.2 = add vp<[[VP14]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.3 = add vp<[[VP15]]>, ir<1>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
+; AFTER-UNROLL-NEXT: vp<[[VP17:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-UNROLL-NEXT: vp<[[VP18:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP16]]>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = concat-vector-parts ir<%add>, ir<%add>.1
+; AFTER-UNROLL-NEXT: WIDEN store x2 vp<[[VP17]]>, vp<[[VP19]]>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = concat-vector-parts ir<%add>.2, ir<%add>.3
+; AFTER-UNROLL-NEXT: WIDEN store x2 vp<[[VP18]]>, vp<[[VP20]]>
+; AFTER-UNROLL-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
+; AFTER-UNROLL-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; AFTER-UNROLL-NEXT: No successors
+; AFTER-UNROLL-NEXT: }
+; AFTER-UNROLL-NEXT: Successor(s): middle.block
+; AFTER-UNROLL-EMPTY:
+; AFTER-UNROLL-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %ptr = getelementptr inbounds i64, ptr %x, i64 %iv
+ %ld = load i64, ptr %ptr, align 8
+ %add = add i64 %ld, 1
+ store i64 %add, ptr %ptr, align 8
+ %next = add nuw i64 %iv, 1
+ %cmp = icmp ult i64 %next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 5fd844e186e44..cb492acc29e1e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -71,6 +71,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printOptimizedVPlan
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMinimumIterationCheck
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceWideCanonicalIVWithWideIV
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::scaleMemoryAccessesByUF
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::unrollByUF
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializePacksAndUnpacks
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeBroadcasts
>From d904ae51e06a3818dc33e3672a6825828b91b0f1 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 20 Aug 2026 19:33:58 +0000
Subject: [PATCH 02/21] Rebase fixups
---
.../AArch64/multi-vector-mem-ops.ll | 20 ++++++++-----------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index f20e7aff70ef7..416eea04db1cb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -17,8 +17,7 @@ define void @mixed_i64_i32_accesses(
; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
@@ -57,7 +56,7 @@ define void @mixed_i64_i32_accesses(
; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP30]], <vscale x 4 x i32> [[TMP27]], i64 8)
; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP31]], <vscale x 4 x i32> [[TMP28]], i64 12)
; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP32]], ptr [[TMP20]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
@@ -98,8 +97,7 @@ define void @mixed_i32_more_frequent_than_i64(
; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
@@ -153,7 +151,7 @@ define void @mixed_i32_more_frequent_than_i64(
; UNMASKED-SVE2P1-NEXT: [[TMP44:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP43]], <vscale x 4 x i32> [[TMP40]], i64 8)
; UNMASKED-SVE2P1-NEXT: [[TMP45:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP44]], <vscale x 4 x i32> [[TMP41]], i64 12)
; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP45]], ptr [[TMP33]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP46]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
@@ -257,8 +255,7 @@ define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 3
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
@@ -277,7 +274,7 @@ define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP6]]
; UNMASKED-SVE2P1-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP7]]
; UNMASKED-SVE2P1-NEXT: [[TMP12]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP8]]
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
@@ -308,8 +305,7 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 3
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[THRESHOLD]], i64 0
; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
@@ -350,7 +346,7 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: [[TMP29]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI1]]
; UNMASKED-SVE2P1-NEXT: [[TMP30]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI2]]
; UNMASKED-SVE2P1-NEXT: [[TMP31]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[VEC_PHI3]]
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
>From 072709bd20de923314eceedb2c5535ca9365987d Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 2 Sep 2026 10:46:03 +0000
Subject: [PATCH 03/21] Fixups
---
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 9fa53493c7d20..73d42be908afb 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5986,8 +5986,7 @@ AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataTy,
if (!ST->enableSubRegLiveness())
return 1;
- if ((Opcode != Instruction::Load && Opcode != Instruction::Store) ||
- !ST->hasSVE2p1() || !VF.isScalable() || !isPowerOf2_32(UF))
+ if (!ST->hasSVE2p1() || !VF.isScalable() || !isPowerOf2_32(UF))
return 1;
unsigned VectorWidth = VF.getKnownMinValue() * DL.getTypeSizeInBits(DataTy);
>From 4a276adbced8b2661b1b6d30518ab418d6aca184 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 2 Sep 2026 13:43:35 +0000
Subject: [PATCH 04/21] Check for extending loads
---
.../llvm/Analysis/TargetTransformInfo.h | 10 +--
.../llvm/Analysis/TargetTransformInfoImpl.h | 8 +--
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 14 ++--
.../AArch64/AArch64TargetTransformInfo.h | 7 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 10 ++-
.../AArch64/multi-vector-mem-ops.ll | 70 +++++++++++++++++++
7 files changed, 104 insertions(+), 19 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 6795267dad097..ed0be2c8ede7a 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1003,12 +1003,14 @@ class TargetTransformInfo {
/// Return the preferred multiple of VF to use for a contiguous load/store.
/// Returning 1 leaves the operation at VF. The returned value must divide UF.
+ /// \p CastHint is non-null if the stored or loaded valued is produced by or
+ /// consumed a cast instruction respectively.
///
/// \p Opcode must be either Instruction::Load or Instruction::Store.
- LLVM_ABI unsigned
- getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataType,
- ElementCount VF, unsigned UF,
- bool IsMasked = false) const;
+ LLVM_ABI unsigned getPreferredVFMultipleForMemoryOp(
+ unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
+ bool IsMasked = false,
+ std::optional<Instruction::CastOps> CastHint = std::nullopt) const;
/// Return true if we should be enabling ordered reductions for the target.
LLVM_ABI bool enableOrderedReductions() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 01006cfce4d08..f70257b10fbac 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -429,11 +429,9 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
- virtual unsigned getPreferredVFMultipleForMemoryOp(unsigned Opcode,
- Type *DataType,
- ElementCount VF,
- unsigned UF,
- bool IsMasked) const {
+ virtual unsigned getPreferredVFMultipleForMemoryOp(
+ unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
+ bool IsMasked, std::optional<Instruction::CastOps> CastHint) const {
return 1;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 05cd06e173200..a54c1fbdc5ffb 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -556,9 +556,9 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
unsigned TargetTransformInfo::getPreferredVFMultipleForMemoryOp(
unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
- bool IsMasked) const {
+ bool IsMasked, std::optional<Instruction::CastOps> CastHint) const {
return TTIImpl->getPreferredVFMultipleForMemoryOp(Opcode, DataType, VF, UF,
- IsMasked);
+ IsMasked, CastHint);
}
bool TargetTransformInfo::isLegalInterleavedAccessType(
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 73d42be908afb..29b5719a09fb0 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5974,15 +5974,21 @@ bool AArch64TTIImpl::isLegalSpeculativeLoad(Type *DataType,
Size.getFixedValue() <= 16;
}
-unsigned
-AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataTy,
- ElementCount VF, unsigned UF,
- bool IsMasked) const {
+unsigned AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(
+ unsigned Opcode, Type *DataTy, ElementCount VF, unsigned UF, bool IsMasked,
+ std::optional<Instruction::CastOps> CastHint) const {
assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
"expected load/store opcode");
if (IsMasked)
return 1; // TODO: Support masked multi-vector loads/stores.
+ // Conservatively, avoid using multi-vector loads when it's possible we could
+ // use extending loads instead. Note: We can ignore stores as we only use
+ // truncating stores when the store vector-width is < a full SVE vector.
+ if (Opcode == Instruction::Load &&
+ (CastHint == Instruction::ZExt || CastHint == Instruction::SExt))
+ return 1;
+
if (!ST->enableSubRegLiveness())
return 1;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index a5513114bd112..aa1d059406c8c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -282,9 +282,10 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isLegalSpeculativeLoad(Type *DataType,
unsigned AddressSpace) const override;
- unsigned getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataType,
- ElementCount VF, unsigned UF,
- bool IsMasked) const override;
+ unsigned getPreferredVFMultipleForMemoryOp(
+ unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
+ bool IsMasked,
+ std::optional<Instruction::CastOps> CastHint) const override;
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 52235e7b7ebb8..bed15eede7d3e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4228,8 +4228,16 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
unsigned Opcode = isa<VPWidenLoadRecipe>(MemOp->getAsRecipe())
? Instruction::Load
: Instruction::Store;
+
+ std::optional<Instruction::CastOps> CastHint;
+ VPUser *MaybeCast = Opcode == Instruction::Store
+ ? StoredValue->getDefiningRecipe()
+ : R.getVPSingleValue()->getSingleUser();
+ if (auto *Cast = dyn_cast_if_present<VPWidenCastRecipe>(MaybeCast))
+ CastHint = Cast->getOpcode();
+
unsigned ScaleFactor = TTI.getPreferredVFMultipleForMemoryOp(
- Opcode, AccessType, VF, UF, /*IsMasked=*/false);
+ Opcode, AccessType, VF, UF, /*IsMasked=*/false, CastHint);
assert((ScaleFactor != 0 && UF % ScaleFactor == 0) &&
"ScaleFactor must divide UF");
MemOp->setVFMultiple(ScaleFactor);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index 416eea04db1cb..27aa10c8d6023 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -368,3 +368,73 @@ loop:
exit:
ret i64 %select.data
}
+
+; Negative test: We should not use a wide load when the result of the load is extended.
+; In this case, it's better to use SVE extending loads (rather than a multi-vector load).
+define void @extending_load(ptr noalias %dst, ptr %src, i64 %n) {
+; UNMASKED-SVE2P1-LABEL: define void @extending_load(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = mul nuw nsw i64 [[TMP2]], 3
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP5]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD2]] to <vscale x 4 x i64>
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD3]] to <vscale x 4 x i64>
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD4]] to <vscale x 4 x i64>
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = mul nsw <vscale x 4 x i64> [[TMP9]], splat (i64 42)
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = mul nsw <vscale x 4 x i64> [[TMP10]], splat (i64 42)
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = mul nsw <vscale x 4 x i64> [[TMP11]], splat (i64 42)
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = mul nsw <vscale x 4 x i64> [[TMP12]], splat (i64 42)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[DST]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP17]], i64 [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP13]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP19]], <vscale x 4 x i64> [[TMP14]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP20]], ptr [[TMP17]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP15]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP21]], <vscale x 4 x i64> [[TMP16]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP22]], ptr [[TMP18]], align 8
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src.ptr = getelementptr inbounds nuw i32, ptr %src, i64 %iv
+ %src.val = load i32, ptr %src.ptr, align 4
+ %conv = sext i32 %src.val to i64
+ %mul = mul nsw i64 %conv, 42
+ %dst.ptr = getelementptr inbounds nuw i64, ptr %dst, i64 %iv
+ store i64 %mul, ptr %dst.ptr, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit.cmp = icmp eq i64 %iv.next, %n
+ br i1 %exit.cmp, label %exit, label %loop
+
+exit:
+ ret void
+}
>From d92d379f1954fa32f51179d93004fe9fb7c0e342 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 3 Sep 2026 16:04:44 +0000
Subject: [PATCH 05/21] Fixups
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 12 ------------
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 9 ++-------
3 files changed, 3 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c0a8007e25192..201f023de7fa6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5793,7 +5793,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
RUN_VPLAN_PASS(VPlanTransforms::scaleMemoryAccessesByUF, BestVPlan, BestVF,
- BestUF, CM.TTI);
+ BestUF, TTI);
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 22f35f1b96ecd..16addfbf1b160 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -293,7 +293,6 @@ struct Recipe_match {
if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
std::is_same_v<RecipeTy, VPVectorEndPointerRecipe> ||
- std::is_same_v<RecipeTy, VPVectorPointerRecipe> ||
std::is_same_v<RecipeTy, VPWidenLoadRecipe> ||
std::is_same_v<RecipeTy, VPWidenStoreRecipe>)
return DefR;
@@ -1052,17 +1051,6 @@ m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
}
-template <typename Op0_t, typename Op1_t>
-using VectorPointerRecipe_match =
- Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
- /*Commutative*/ false, VPVectorPointerRecipe>;
-
-template <typename Op0_t, typename Op1_t>
-VectorPointerRecipe_match<Op0_t, Op1_t> m_VecPtr(const Op0_t &Op0,
- const Op1_t &Op1) {
- return VectorPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
-}
-
template <typename Op0_t>
using VPWidenLoadRecipe_match =
Recipe_match<std::tuple<Op0_t>, 0,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bed15eede7d3e..4cfc371853235 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4205,14 +4205,9 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- uint64_t Stride;
VPValue *StoredValue = nullptr;
- auto m_ConstantStrideVecPtr =
- m_VecPtr(m_VPValue(), m_ConstantInt(Stride));
- if ((!match(&R, m_WidenLoad(m_ConstantStrideVecPtr)) &&
- !match(&R, m_WidenStore(m_ConstantStrideVecPtr,
- m_VPValue(StoredValue)))) ||
- Stride != 1)
+ if (!match(&R, m_WidenLoad(m_VPValue())) &&
+ !match(&R, m_WidenStore(m_VPValue(), m_VPValue(StoredValue))))
continue;
auto *MemOp = cast<VPWidenMemoryRecipe>(&R);
>From 1bfbccd2375e372a29d2868f630e8253e31ed1c1 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 7 Sep 2026 11:20:54 +0000
Subject: [PATCH 06/21] Fixups
---
.../Transforms/Vectorize/VPlanPatternMatch.h | 12 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 22 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 12 +-
...-vector-mem-ops-non-power-of-two-unroll.ll | 74 +++
.../AArch64/multi-vector-mem-ops.ll | 445 ++++++++++++------
5 files changed, 411 insertions(+), 154 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 16addfbf1b160..22f35f1b96ecd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -293,6 +293,7 @@ struct Recipe_match {
if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
std::is_same_v<RecipeTy, VPVectorEndPointerRecipe> ||
+ std::is_same_v<RecipeTy, VPVectorPointerRecipe> ||
std::is_same_v<RecipeTy, VPWidenLoadRecipe> ||
std::is_same_v<RecipeTy, VPWidenStoreRecipe>)
return DefR;
@@ -1051,6 +1052,17 @@ m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
}
+template <typename Op0_t, typename Op1_t>
+using VectorPointerRecipe_match =
+ Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
+ /*Commutative*/ false, VPVectorPointerRecipe>;
+
+template <typename Op0_t, typename Op1_t>
+VectorPointerRecipe_match<Op0_t, Op1_t> m_VecPtr(const Op0_t &Op0,
+ const Op1_t &Op1) {
+ return VectorPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
+}
+
template <typename Op0_t>
using VPWidenLoadRecipe_match =
Recipe_match<std::tuple<Op0_t>, 0,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4cfc371853235..63b9cfa05f85e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4203,16 +4203,20 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
return;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
+ vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ uint64_t Stride;
VPValue *StoredValue = nullptr;
- if (!match(&R, m_WidenLoad(m_VPValue())) &&
- !match(&R, m_WidenStore(m_VPValue(), m_VPValue(StoredValue))))
+ auto m_ConstantStrideVecPtr =
+ m_VecPtr(m_VPValue(), m_ConstantInt(Stride));
+ if ((!match(&R, m_WidenLoad(m_ConstantStrideVecPtr)) &&
+ !match(&R, m_WidenStore(m_ConstantStrideVecPtr,
+ m_VPValue(StoredValue)))) ||
+ Stride != 1)
continue;
auto *MemOp = cast<VPWidenMemoryRecipe>(&R);
- if (!MemOp->isConsecutive())
- continue;
+ assert(MemOp->isConsecutive() && "Expected consecutive load/store");
// TODO: Support masked loads/stores. This requires widening the header
// mask to the same factor as the memory operation.
@@ -4231,11 +4235,11 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
if (auto *Cast = dyn_cast_if_present<VPWidenCastRecipe>(MaybeCast))
CastHint = Cast->getOpcode();
- unsigned ScaleFactor = TTI.getPreferredVFMultipleForMemoryOp(
+ unsigned VFMultiple = TTI.getPreferredVFMultipleForMemoryOp(
Opcode, AccessType, VF, UF, /*IsMasked=*/false, CastHint);
- assert((ScaleFactor != 0 && UF % ScaleFactor == 0) &&
- "ScaleFactor must divide UF");
- MemOp->setVFMultiple(ScaleFactor);
+ assert((VFMultiple != 0 && UF % VFMultiple == 0) &&
+ "VFMultiple must divide UF");
+ MemOp->setVFMultiple(VFMultiple);
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 29b33f607952e..9de8a0e99358c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -295,7 +295,8 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
void UnrollState::unrollMemOpWithVFMultiple(VPRecipeBase &R,
unsigned VFMultiple) {
- assert(VFMultiple > 1 && UF % VFMultiple == 0);
+ assert(VFMultiple > 1 && UF % VFMultiple == 0 &&
+ "expected VFMultiple to divide UF");
SmallVector<VPRecipeBase *, 4> Groups(UF / VFMultiple, nullptr);
Groups[0] = &R;
@@ -361,10 +362,11 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
}
}
- if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(&R);
- WidenMem && WidenMem->getVFMultiple() > 1) {
- unrollMemOpWithVFMultiple(R, WidenMem->getVFMultiple());
- return;
+ if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(&R)) {
+ if (WidenMem && WidenMem->getVFMultiple() > 1) {
+ unrollMemOpWithVFMultiple(R, WidenMem->getVFMultiple());
+ return;
+ }
}
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
new file mode 100644
index 0000000000000..ac9e55117364e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-interleave=3 -aarch64-enable-subreg-liveness-tracking -sve-tail-folding=disabled %s | FileCheck %s --check-prefix=CHECK
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Tests `scaleMemoryAccessesByUF` with a non-power-of-two unroll factor.
+; On AArch64, this should be rejected (which results in `vscale x 4` loads/stores).
+
+define void @mixed_i64_i32_accesses(ptr noalias %x, ptr noalias %y, i64 %n) {
+; CHECK-LABEL: define void @mixed_i64_i32_accesses(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 12
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP5]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i64> [[WIDE_LOAD]], splat (i64 1)
+; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i64> [[WIDE_LOAD1]], splat (i64 1)
+; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[WIDE_LOAD2]], splat (i64 1)
+; CHECK-NEXT: store <vscale x 4 x i64> [[TMP8]], ptr [[TMP4]], align 8
+; CHECK-NEXT: store <vscale x 4 x i64> [[TMP9]], ptr [[TMP6]], align 8
+; CHECK-NEXT: store <vscale x 4 x i64> [[TMP10]], ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i64 [[TMP5]]
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 4 x i32>, ptr [[TMP13]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD3]], splat (i32 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD4]], splat (i32 1)
+; CHECK-NEXT: [[TMP16:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD5]], splat (i32 1)
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP14]], ptr [[TMP11]], align 4
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP15]], ptr [[TMP12]], align 4
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP16]], ptr [[TMP13]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %px = getelementptr inbounds i64, ptr %x, i64 %iv
+ %vx = load i64, ptr %px, align 8
+ %ax = add i64 %vx, 1
+ store i64 %ax, ptr %px, align 8
+ %py = getelementptr inbounds i32, ptr %y, i64 %iv
+ %vy = load i32, ptr %py, align 4
+ %ay = add i32 %vy, 1
+ store i32 %ay, ptr %py, align 4
+ %next = add nuw i64 %iv, 1
+ %cmp = icmp ult i64 %next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index 27aa10c8d6023..b90e29519e24e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -3,30 +3,30 @@
target triple = "aarch64-unknown-linux-gnu"
-define void @mixed_i64_i32_accesses(
+define void @mixed_i64_i32_accesses(ptr noalias %x, ptr noalias %y, i64 %n) {
; UNMASKED-SVE2P1-LABEL: define void @mixed_i64_i32_accesses(
; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[TMP3]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i64 [[TMP14]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i64>, ptr [[TMP15]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 0)
@@ -35,33 +35,32 @@ define void @mixed_i64_i32_accesses(
; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP15]], ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP13]], i64 4)
; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP17]], ptr [[TMP5]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP13]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP19]], ptr [[TMP15]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i32>, ptr [[TMP20]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = add <vscale x 4 x i32> [[TMP23]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = add <vscale x 4 x i32> [[TMP24]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP25]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP30]], <vscale x 4 x i32> [[TMP27]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP31]], <vscale x 4 x i32> [[TMP28]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP32]], ptr [[TMP20]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP24]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP25]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP30]], ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
- ptr noalias %x, ptr noalias %y, i64 %n) {
entry:
br label %loop
@@ -83,80 +82,79 @@ exit:
ret void
}
-define void @mixed_i32_more_frequent_than_i64(
+define void @mixed_i32_more_frequent_than_i64(ptr noalias %x, ptr noalias %y, ptr noalias %z, i64 %n) {
; UNMASKED-SVE2P1-LABEL: define void @mixed_i32_more_frequent_than_i64(
; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[TMP3]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i64 [[TMP14]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i64>, ptr [[TMP15]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD3]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD3]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 4)
; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP15]], ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP13]], i64 4)
; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP17]], ptr [[TMP5]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP13]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP19]], ptr [[TMP15]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i32>, ptr [[TMP20]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD1]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = add <vscale x 4 x i32> [[TMP23]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = add <vscale x 4 x i32> [[TMP24]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP25]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP30]], <vscale x 4 x i32> [[TMP27]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP31]], <vscale x 4 x i32> [[TMP28]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP32]], ptr [[TMP20]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i32>, ptr [[TMP33]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP36:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP37:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD2]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP24]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP25]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP30]], ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i32>, ptr [[TMP31]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP36:%.*]] = add <vscale x 4 x i32> [[TMP32]], splat (i32 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[TMP33]], splat (i32 2)
; UNMASKED-SVE2P1-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], splat (i32 2)
; UNMASKED-SVE2P1-NEXT: [[TMP39:%.*]] = add <vscale x 4 x i32> [[TMP35]], splat (i32 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP40:%.*]] = add <vscale x 4 x i32> [[TMP36]], splat (i32 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP41:%.*]] = add <vscale x 4 x i32> [[TMP37]], splat (i32 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP42:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP38]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP43:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP42]], <vscale x 4 x i32> [[TMP39]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP44:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP43]], <vscale x 4 x i32> [[TMP40]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP45:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP44]], <vscale x 4 x i32> [[TMP41]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP45]], ptr [[TMP33]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP46]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: [[TMP40:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP36]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP41:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP40]], <vscale x 4 x i32> [[TMP37]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP42:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP41]], <vscale x 4 x i32> [[TMP38]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP43:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP42]], <vscale x 4 x i32> [[TMP39]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP43]], ptr [[TMP31]], align 4
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
- ptr noalias %x, ptr noalias %y, ptr noalias %z, i64 %n) {
entry:
br label %loop
@@ -182,7 +180,7 @@ exit:
ret void
}
-define void @first_order_recurrence_i64_scaled_load_and_store(
+define void @first_order_recurrence_i64_scaled_load_and_store(ptr noalias %src, ptr noalias %dst) {
; UNMASKED-SVE2P1-LABEL: define void @first_order_recurrence_i64_scaled_load_and_store(
; UNMASKED-SVE2P1-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
@@ -224,7 +222,6 @@ define void @first_order_recurrence_i64_scaled_load_and_store(
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
- ptr noalias %src, ptr noalias %dst) {
entry:
%first = load i64, ptr %src, align 8
br label %loop
@@ -260,23 +257,23 @@ define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP4]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
-; UNMASKED-SVE2P1-NEXT: [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP5]]
-; UNMASKED-SVE2P1-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP6]]
-; UNMASKED-SVE2P1-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP7]]
-; UNMASKED-SVE2P1-NEXT: [[TMP12]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP8]]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
+; UNMASKED-SVE2P1-NEXT: [[TMP8]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP5]]
+; UNMASKED-SVE2P1-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP6]]
+; UNMASKED-SVE2P1-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP7]]
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -312,43 +309,43 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP7]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP6]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP7]]
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP8]]
; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP11]]
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = freeze <vscale x 2 x i1> [[TMP11]]
; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = freeze <vscale x 2 x i1> [[TMP12]]
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = freeze <vscale x 2 x i1> [[TMP13]]
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = or <vscale x 2 x i1> [[TMP16]], [[TMP17]]
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = freeze <vscale x 2 x i1> [[TMP14]]
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = or <vscale x 2 x i1> [[TMP18]], [[TMP19]]
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = freeze <vscale x 2 x i1> [[TMP15]]
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = or <vscale x 2 x i1> [[TMP20]], [[TMP21]]
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP22]])
-; UNMASKED-SVE2P1-NEXT: [[TMP24]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> [[TMP3]]
-; UNMASKED-SVE2P1-NEXT: [[TMP25]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP13]], <vscale x 2 x i1> [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP26]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> [[TMP5]]
-; UNMASKED-SVE2P1-NEXT: [[TMP27]] = select i1 [[TMP23]], <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP6]]
-; UNMASKED-SVE2P1-NEXT: [[TMP28]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[VEC_PHI]]
-; UNMASKED-SVE2P1-NEXT: [[TMP29]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP30]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP31]] = select i1 [[TMP23]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[VEC_PHI3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = or <vscale x 2 x i1> [[TMP15]], [[TMP16]]
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = freeze <vscale x 2 x i1> [[TMP13]]
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = or <vscale x 2 x i1> [[TMP17]], [[TMP18]]
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = freeze <vscale x 2 x i1> [[TMP14]]
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = or <vscale x 2 x i1> [[TMP19]], [[TMP20]]
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP21]])
+; UNMASKED-SVE2P1-NEXT: [[TMP23]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[TMP24]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP25]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP13]], <vscale x 2 x i1> [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP26]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> [[TMP5]]
+; UNMASKED-SVE2P1-NEXT: [[TMP27]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[VEC_PHI]]
+; UNMASKED-SVE2P1-NEXT: [[TMP28]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[VEC_PHI1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP29]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI2]]
+; UNMASKED-SVE2P1-NEXT: [[TMP30]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI3]]
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -438,3 +435,171 @@ loop:
exit:
ret void
}
+
+; TODO: Support multi-vector operations with negative strides.
+define void @reverse_stride(ptr noalias %c, ptr noalias %a, ptr noalias %b, i64 %n) {
+; UNMASKED-SVE2P1-LABEL: define void @reverse_stride(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[C:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; UNMASKED-SVE2P1-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[N]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SMIN]]
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 2
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = sub i64 [[N]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP6]]
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = sub nuw nsw i64 [[TMP4]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = sub i64 0, [[TMP8]]
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP9]]
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = sub i64 [[TMP9]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP27]]
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = mul i64 -2, [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = sub i64 [[TMP29]], [[TMP8]]
+; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP33]]
+; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = mul i64 -3, [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP36:%.*]] = sub i64 [[TMP35]], [[TMP8]]
+; UNMASKED-SVE2P1-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP36]]
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = load <vscale x 4 x i32>, ptr [[TMP28]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = load <vscale x 4 x i32>, ptr [[TMP34]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = load <vscale x 4 x i32>, ptr [[TMP37]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP6]]
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP9]]
+; UNMASKED-SVE2P1-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP27]]
+; UNMASKED-SVE2P1-NEXT: [[TMP39:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP33]]
+; UNMASKED-SVE2P1-NEXT: [[TMP40:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP36]]
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 4 x i32>, ptr [[TMP16]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = load <vscale x 4 x i32>, ptr [[TMP38]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = load <vscale x 4 x i32>, ptr [[TMP39]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = load <vscale x 4 x i32>, ptr [[TMP40]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = add nsw <vscale x 4 x i32> [[TMP17]], [[TMP11]]
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = add nsw <vscale x 4 x i32> [[TMP18]], [[TMP12]]
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add nsw <vscale x 4 x i32> [[TMP19]], [[TMP13]]
+; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add nsw <vscale x 4 x i32> [[TMP20]], [[TMP14]]
+; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[TMP6]]
+; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP9]]
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP27]]
+; UNMASKED-SVE2P1-NEXT: [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP33]]
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP36]]
+; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP21]], ptr [[TMP26]], align 4
+; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP22]], ptr [[TMP30]], align 4
+; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP23]], ptr [[TMP41]], align 4
+; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP24]], ptr [[TMP32]], align 4
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ %dec, %for.body ], [ %n, %entry ]
+ %a.ptr = getelementptr inbounds nuw i32, ptr %a, i64 %iv
+ %a.val = load i32, ptr %a.ptr, align 4
+ %b.ptr = getelementptr inbounds nuw i32, ptr %b, i64 %iv
+ %b.val = load i32, ptr %b.ptr, align 4
+ %add = add nsw i32 %b.val, %a.val
+ %c.ptr = getelementptr inbounds nuw i32, ptr %c, i64 %iv
+ store i32 %add, ptr %c.ptr, align 4
+ %dec = add nsw i64 %iv, -1
+ %cmp = icmp sgt i64 %iv, 0
+ br i1 %cmp, label %for.body, label %exit
+
+exit:
+ ret void
+}
+
+; Negative test (load): A strided load (stride = 2). Only the store should be widened to a VF multiple.
+define void @gather_nxv4i32_stride2(ptr noalias %a, ptr noalias %b, i64 %n) {
+; UNMASKED-SVE2P1-LABEL: define void @gather_nxv4i32_stride2(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[TMP1]], i64 [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = add i64 [[TMP2]], 0
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = shl i64 [[TMP2]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = mul i64 [[TMP2]], 3
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 0
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = shl i64 [[TMP7]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = shl i64 [[TMP11]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = shl i64 [[TMP15]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP16]]
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]]
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]]
+; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
+; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC4:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
+; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC5:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC4]])
+; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC5]], 0
+; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC6:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
+; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC6]])
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC7]], 0
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP24]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP29]], <vscale x 4 x float> [[TMP25]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP30]], <vscale x 4 x float> [[TMP26]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP31]], <vscale x 4 x float> [[TMP27]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x float> [[TMP32]], ptr [[TMP28]], align 4
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+ %iv.stride2 = mul i64 %iv, 2
+ %b.ptr = getelementptr inbounds float, ptr %b, i64 %iv.stride2
+ %b.val = load float, ptr %b.ptr, align 4
+ %a.ptr = getelementptr inbounds float, ptr %a, i64 %iv
+ store float %b.val, ptr %a.ptr, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
>From f74fbce6a3117bd18792659b6da16d921ca59cd0 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 7 Sep 2026 13:14:38 +0000
Subject: [PATCH 07/21] Rework opcodes
---
llvm/lib/Transforms/Vectorize/VPlan.h | 38 +++++------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 55 ++++++++++------
.../Transforms/Vectorize/VPlanTransforms.cpp | 27 +++++++-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 65 ++++++++++---------
.../vplan-printing-multi-vector-mem-ops.ll | 40 ++++++------
5 files changed, 131 insertions(+), 94 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0edf821a0806f..dcf15a8c3102d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1317,10 +1317,18 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// WideActiveLaneMask is used for control flow and is unrolled by widening,
// with one extract vector created per unroll part.
WideActiveLaneMask,
+ // Signature: (VFMultiple, Address, Alignment) -> Wide Vector
+ // Loads a single wide vector of `VFMultiple * VF` elements. VFMultiple must
+ // divide UF. After unrolling, each section of VF elements in the wide
+ // vector corresponds to an unroll part.
+ VFMultipleLoad,
+ // Signature: (VFMultiple, Address, Alignment, Vectors...)
+ // Concatenates VFMultiple vector operands into a single wide vector of
+ // `VFMultiple * VF` elements and stores it. After unrolling, each vector
+ // operand corresponds to an unroll part.
+ VFMultipleStore,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
- // Concatenates its unrolled part operands into one widened vector.
- ConcatVectorParts,
ExplicitVectorLength,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
@@ -1523,6 +1531,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
case VPInstruction::BranchOnCond:
case VPInstruction::BranchOnTwoConds:
case VPInstruction::BranchOnCount:
+ case VPInstruction::VFMultipleStore:
return false;
default:
return true;
@@ -3777,10 +3786,6 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
/// Whether the memory access is masked.
bool IsMasked = false;
- /// Multiple of VF used to widen this memory operation. The final operation
- /// loads or stores VF * VFMultiple elements
- unsigned VFMultiple = 1;
-
void setMask(VPValue *Mask) {
assert(!IsMasked && "cannot re-set mask");
if (!Mask)
@@ -3827,12 +3832,6 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
Instruction &getIngredient() const { return Ingredient; }
-
- /// Set the VF multiple for this memory operation.
- void setVFMultiple(unsigned VFMultiple) { this->VFMultiple = VFMultiple; }
-
- /// Returns the VF multiple of this memory operation.
- unsigned getVFMultiple() const { return VFMultiple; }
};
/// A recipe for widening load operations, using the address to load from and an
@@ -3848,11 +3847,8 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPSingleDefRecipe,
}
VPWidenLoadRecipe *clone() override {
- auto *R =
- new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(), getMask(),
- Consecutive, *this, getDebugLoc());
- R->setVFMultiple(VFMultiple);
- return R;
+ return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
+ getMask(), Consecutive, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadSC);
@@ -3956,11 +3952,9 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPRecipeBase,
}
VPWidenStoreRecipe *clone() override {
- auto *R = new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
- getStoredValue(), getMask(), Consecutive,
- *this, getDebugLoc());
- R->setVFMultiple(VFMultiple);
- return R;
+ return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
+ getStoredValue(), getMask(), Consecutive,
+ *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStoreSC);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 6af97a147f239..bba60d7c009dd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -491,6 +491,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
for (unsigned Idx = 1; Idx != Operands.size(); ++Idx)
AssertOperandType(Idx, Op0Ty);
return Type::getVoidTy(Ctx);
+ case VPInstruction::VFMultipleStore:
case Instruction::Store:
return Type::getVoidTy(Ctx);
case Instruction::ICmp:
@@ -564,9 +565,9 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
return StructTy->getTypeAtIndex(
cast<VPConstantInt>(Operands[1])->getZExtValue());
}
- case VPInstruction::ConcatVectorParts:
case VPInstruction::ExtractVectorForPart:
return Op0Ty;
+ case VPInstruction::VFMultipleLoad:
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::NumActiveLanes:
@@ -675,6 +676,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::Select:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
+ case VPInstruction::VFMultipleLoad:
return 3;
case Instruction::Call:
return getCalledFnOperandIndex(operands()) + 1;
@@ -694,7 +696,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLane:
case VPInstruction::ExtractLastActive:
- case VPInstruction::ConcatVectorParts:
+ case VPInstruction::VFMultipleStore:
// Cannot determine the number of operands from the opcode.
return -1u;
}
@@ -1184,18 +1186,32 @@ Value *VPInstruction::generate(VPTransformState &State,
vputils::getIntrinsicID(this), Args,
/*FMFSource=*/nullptr, getName());
}
- case VPInstruction::ConcatVectorParts: {
- unsigned VectorOps = getNumOperands();
+ case VPInstruction::VFMultipleLoad: {
+ unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
auto *WideDataTy = VectorType::get(
- getScalarType(), State.VF.multiplyCoefficientBy(VectorOps));
- Value *WideData = PoisonValue::get(WideDataTy);
+ getScalarType(), State.VF.multiplyCoefficientBy(VFMultiple));
+
+ Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
+ Align Alignment = Align(cast<VPConstantInt>(getOperand(2))->getZExtValue());
+ return Builder.CreateAlignedLoad(WideDataTy, Addr, Alignment,
+ "vf.multiple.load");
+ }
+ case VPInstruction::VFMultipleStore: {
+ unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
+ Type *ScalarStoreTy = getOperand(3)->getScalarType();
+ auto *WideDataTy = VectorType::get(
+ ScalarStoreTy, State.VF.multiplyCoefficientBy(VFMultiple));
- for (unsigned I = 0; I < VectorOps; ++I) {
- Value *Part = State.get(getOperand(I));
+ Value *WideData = PoisonValue::get(WideDataTy);
+ for (unsigned I = 0; I < VFMultiple; ++I) {
+ Value *Part = State.get(getOperand(I + 3));
WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part,
I * State.VF.getKnownMinValue());
}
- return WideData;
+
+ Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
+ Align Alignment = Align(cast<VPConstantInt>(getOperand(2))->getZExtValue());
+ return Builder.CreateAlignedStore(WideData, Addr, Alignment);
}
default:
llvm_unreachable("Unsupported opcode for instruction");
@@ -1680,6 +1696,10 @@ void VPInstruction::addOperand(VPValue *Op) {
"matching operand 1's type and i1, respectively");
break;
}
+ case VPInstruction::VFMultipleStore:
+ assert(Ty == getOperand(3)->getScalarType() &&
+ "appended operand must match operand 3's scalar type");
+ break;
default:
llvm_unreachable("opcode does not support growing the operand list "
"outside of construction");
@@ -1746,7 +1766,6 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::IncomingAliasMask:
- case VPInstruction::ConcatVectorParts:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::FirstActiveLane:
@@ -1880,12 +1899,15 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::WideActiveLaneMask:
O << "wide active lane mask";
break;
+ case VPInstruction::VFMultipleLoad:
+ O << "vf-multiple load";
+ break;
+ case VPInstruction::VFMultipleStore:
+ O << "vf-multiple store";
+ break;
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
break;
- case VPInstruction::ConcatVectorParts:
- O << "concat-vector-parts";
- break;
case VPInstruction::ExplicitVectorLength:
O << "EXPLICIT-VECTOR-LENGTH";
break;
@@ -4394,8 +4416,7 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
void VPWidenLoadRecipe::execute(VPTransformState &State) {
Type *ScalarDataTy = getScalarType();
- auto *DataTy =
- VectorType::get(ScalarDataTy, State.VF.multiplyCoefficientBy(VFMultiple));
+ auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
@@ -4425,8 +4446,6 @@ void VPWidenLoadRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN ";
printAsOperand(O, SlotTracker);
O << " = load ";
- if (VFMultiple > 1)
- O << "x" << VFMultiple << ' ';
printOperands(O, SlotTracker);
}
#endif
@@ -4514,8 +4533,6 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
void VPWidenStoreRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN store ";
- if (VFMultiple > 1)
- O << "x" << VFMultiple << ' ';
printOperands(O, SlotTracker);
}
#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 63b9cfa05f85e..69ab111ff3e1d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4202,6 +4202,8 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
if (UF == 1)
return;
+ Type *IVTy = Plan.getVectorLoopRegion()->getCanonicalIVType();
+
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -4239,7 +4241,30 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
Opcode, AccessType, VF, UF, /*IsMasked=*/false, CastHint);
assert((VFMultiple != 0 && UF % VFMultiple == 0) &&
"VFMultiple must divide UF");
- MemOp->setVFMultiple(VFMultiple);
+
+ if (VFMultiple == 1)
+ continue;
+
+ VPBuilder Builder(VPBB, R.getIterator());
+
+ VPValue *Ptr = MemOp->getAddr();
+ VPValue *VFMultipleVPV = Plan.getConstantInt(IVTy, VFMultiple);
+ VPValue *Align = Plan.getConstantInt(IVTy, MemOp->getAlign().value());
+
+ if (Opcode == Instruction::Load) {
+ VPValue *OldLoad = R.getVPSingleValue();
+ VPValue *Load = Builder.createNaryOp(
+ VPInstruction::VFMultipleLoad, {VFMultipleVPV, Ptr, Align}, nullptr,
+ {}, {}, DebugLoc::getUnknown(), "", OldLoad->getScalarType());
+ OldLoad->replaceAllUsesWith(Load);
+ } else {
+ assert(Opcode == Instruction::Store);
+ Builder.createNaryOp(VPInstruction::VFMultipleStore,
+ {VFMultipleVPV, Ptr, Align, StoredValue},
+ R.getDebugLoc());
+ }
+
+ R.eraseFromParent();
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 9de8a0e99358c..f73392db568d2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -74,8 +74,8 @@ class UnrollState {
return Plan.getConstantInt(CanIVIntTy, Part);
}
- /// Unroll a VPWidenLoadRecipe or VPWidenStoreRecipe with a VFMultiple > 1.
- void unrollMemOpWithVFMultiple(VPRecipeBase &R, unsigned VFMultiple);
+ /// Unroll a VFMultipleLoad or VFMultipleStore VPInstruction.
+ void unrollMemOpWithVFMultiple(VPInstruction *VPI);
public:
UnrollState(VPlan &Plan, unsigned UF) : Plan(Plan), UF(UF) {}
@@ -293,53 +293,57 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
}
}
-void UnrollState::unrollMemOpWithVFMultiple(VPRecipeBase &R,
- unsigned VFMultiple) {
+void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
+ assert(VPI->getOpcode() == VPInstruction::VFMultipleLoad ||
+ VPI->getOpcode() == VPInstruction::VFMultipleStore);
+
+ unsigned VFMultiple = cast<VPConstantInt>(VPI->getOperand(0))->getZExtValue();
assert(VFMultiple > 1 && UF % VFMultiple == 0 &&
"expected VFMultiple to divide UF");
- SmallVector<VPRecipeBase *, 4> Groups(UF / VFMultiple, nullptr);
- Groups[0] = &R;
+
+ SmallVector<VPInstruction *, 4> Groups(UF / VFMultiple, nullptr);
+ Groups[0] = VPI;
// A memory op with a VFMultiple is widened to VF * VFMultiple elements, so
// after unrolling by UF we materialize UF / VFMultiple such ops, each
// covering VFMultiple unroll parts.
- VPBuilder Builder = VPBuilder::getToInsertAfter(&R);
+ VPBuilder Builder = VPBuilder::getToInsertAfter(VPI);
for (unsigned Group = 1; Group < Groups.size(); ++Group) {
- auto *Copy = Builder.insert(R.clone());
+ auto *Copy = Builder.insert(VPI->clone());
remapOperands(Copy, Group * VFMultiple);
Groups[Group] = Copy;
}
- if (auto *Store = dyn_cast<VPWidenStoreRecipe>(&R)) {
- VPValue *StoredValue = Store->getStoredValue();
+ if (VPI->getOpcode() == VPInstruction::VFMultipleStore) {
+ VPValue *StoredValue = VPI->getOperand(3);
for (unsigned Group = 0; Group < Groups.size(); ++Group) {
- VPRecipeBase *Store = Groups[Group];
- Builder.setInsertPoint(Store);
- SmallVector<VPValue *, 4> Parts;
- // We need to concatenate VFMultiple parts to form the stored value.
- for (unsigned Part = 0; Part < VFMultiple; ++Part)
- Parts.push_back(
- getValueForPart(StoredValue, Group * VFMultiple + Part));
- auto *Concat =
- Builder.createNaryOp(VPInstruction::ConcatVectorParts, Parts);
- Groups[Group]->setOperand(1, Concat);
- ToSkip.insert(Concat);
+ VPInstruction *Store = Groups[Group];
+ // Add the value to store for each unroll part in this group.
+ for (unsigned Part = 0; Part < VFMultiple; ++Part) {
+ VPValue *UnrollPart =
+ getValueForPart(StoredValue, Group * VFMultiple + Part);
+ if (Part == 0)
+ Store->setOperand(3, UnrollPart);
+ else
+ Store->addOperand(UnrollPart);
+ }
}
} else {
- assert(isa<VPWidenLoadRecipe>(R) && "Expected a load recipe");
+ assert(VPI->getOpcode() == VPInstruction::VFMultipleLoad &&
+ "Expected a load recipe");
// We need to extract each unroll part as a subvector.
auto ExtractPart0 = Builder.createNaryOp(
VPInstruction::ExtractVectorForPart,
{Groups[0]->getVPSingleValue(), getConstantInt(0)});
- // First replace R with an extract of the first unroll part (ExtractPart0).
- R.getVPSingleValue()->replaceUsesWithIf(
+ // First VPI with an extract of the first unroll part (ExtractPart0).
+ VPI->getVPSingleValue()->replaceUsesWithIf(
ExtractPart0, [&](VPUser &U, unsigned) { return &U != ExtractPart0; });
ToSkip.insert(ExtractPart0);
// Create extracts for the remaining unroll parts and remap later uses of
// ExtractPart0 to the correct unrolled part.
for (unsigned Part = 1; Part != UF; ++Part) {
- VPRecipeBase *Group = Groups[Part / VFMultiple];
+ VPInstruction *Group = Groups[Part / VFMultiple];
unsigned IndexInGroup = Part % VFMultiple;
auto *Extract = Builder.createNaryOp(
VPInstruction::ExtractVectorForPart,
@@ -356,15 +360,14 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
return;
if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
- if (vputils::onlyFirstPartUsed(VPI)) {
- addUniformForAllParts(VPI);
+ if (VPI->getOpcode() == VPInstruction::VFMultipleLoad ||
+ VPI->getOpcode() == VPInstruction::VFMultipleStore) {
+ unrollMemOpWithVFMultiple(VPI);
return;
}
- }
- if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(&R)) {
- if (WidenMem && WidenMem->getVFMultiple() > 1) {
- unrollMemOpWithVFMultiple(R, WidenMem->getVFMultiple());
+ if (vputils::onlyFirstPartUsed(VPI)) {
+ addUniformForAllParts(VPI);
return;
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
index 3943cb5f28103..15dc08babf853 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
@@ -69,10 +69,10 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-SCALE-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; AFTER-SCALE-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
; AFTER-SCALE-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-SCALE-NEXT: WIDEN ir<%ld> = load x2 vp<[[VP9]]>
-; AFTER-SCALE-NEXT: WIDEN ir<%add> = add ir<%ld>, ir<1>
-; AFTER-SCALE-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-SCALE-NEXT: WIDEN store x2 vp<[[VP10]]>, ir<%add>
+; AFTER-SCALE-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP9]]>, ir<8>
+; AFTER-SCALE-NEXT: WIDEN ir<%add> = add vp<[[VP10]]>, ir<1>
+; AFTER-SCALE-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-SCALE-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP11]]>, ir<8>, ir<%add>
; AFTER-SCALE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
; AFTER-SCALE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; AFTER-SCALE-NEXT: No successors
@@ -108,23 +108,21 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-UNROLL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
; AFTER-UNROLL-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
; AFTER-UNROLL-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP9]]>
-; AFTER-UNROLL-NEXT: WIDEN ir<%ld> = load x2 vp<[[VP10]]>
-; AFTER-UNROLL-NEXT: WIDEN ir<%ld>.1 = load x2 vp<[[VP11]]>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-vector-for-part ir<%ld>, ir<0>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-vector-for-part ir<%ld>, ir<1>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-vector-for-part ir<%ld>.1, ir<0>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-vector-for-part ir<%ld>.1, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add> = add vp<[[VP12]]>, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add>.1 = add vp<[[VP13]]>, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add>.2 = add vp<[[VP14]]>, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add>.3 = add vp<[[VP15]]>, ir<1>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
-; AFTER-UNROLL-NEXT: vp<[[VP17:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-UNROLL-NEXT: vp<[[VP18:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP16]]>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = concat-vector-parts ir<%add>, ir<%add>.1
-; AFTER-UNROLL-NEXT: WIDEN store x2 vp<[[VP17]]>, vp<[[VP19]]>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = concat-vector-parts ir<%add>.2, ir<%add>.3
-; AFTER-UNROLL-NEXT: WIDEN store x2 vp<[[VP18]]>, vp<[[VP20]]>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP10]]>, ir<8>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP11]]>, ir<8>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<0>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<1>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-vector-for-part vp<[[VP13]]>, ir<0>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-vector-for-part vp<[[VP13]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add> = add vp<[[VP14]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.1 = add vp<[[VP15]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.2 = add vp<[[VP16]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.3 = add vp<[[VP17]]>, ir<1>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
+; AFTER-UNROLL-NEXT: vp<[[VP19:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-UNROLL-NEXT: vp<[[VP20:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP18]]>
+; AFTER-UNROLL-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP19]]>, ir<8>, ir<%add>, ir<%add>.1
+; AFTER-UNROLL-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP20]]>, ir<8>, ir<%add>.2, ir<%add>.3
; AFTER-UNROLL-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
; AFTER-UNROLL-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; AFTER-UNROLL-NEXT: No successors
>From 977e69309e69ecf441042834d36166b664507651 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 7 Sep 2026 14:09:53 +0000
Subject: [PATCH 08/21] Fixups
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +++++++++---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 +--
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bba60d7c009dd..19d3b5ed7951c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -59,7 +59,8 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPInstructionSC: {
auto *VPI = cast<VPInstruction>(this);
// Loads read from memory but don't write to memory.
- if (VPI->getOpcode() == Instruction::Load)
+ if (VPI->getOpcode() == Instruction::Load ||
+ VPI->getOpcode() == VPInstruction::VFMultipleLoad)
return false;
return VPI->opcodeMayReadOrWriteFromMemory();
}
@@ -118,8 +119,13 @@ bool VPRecipeBase::mayReadFromMemory() const {
switch (getVPRecipeID()) {
case VPExpressionSC:
return cast<VPExpressionRecipe>(this)->mayReadOrWriteMemory();
- case VPInstructionSC:
- return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory();
+ case VPInstructionSC: {
+ auto *VPI = cast<VPInstruction>(this);
+ // Stores write to memory but don't read from memory.
+ if (VPI->getOpcode() == VPInstruction::VFMultipleStore)
+ return false;
+ return VPI->opcodeMayReadOrWriteFromMemory();
+ }
case VPWidenLoadEVLSC:
case VPWidenLoadSC:
return true;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 69ab111ff3e1d..d87e623892ced 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4245,12 +4245,11 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
if (VFMultiple == 1)
continue;
- VPBuilder Builder(VPBB, R.getIterator());
-
VPValue *Ptr = MemOp->getAddr();
VPValue *VFMultipleVPV = Plan.getConstantInt(IVTy, VFMultiple);
VPValue *Align = Plan.getConstantInt(IVTy, MemOp->getAlign().value());
+ VPBuilder Builder(VPBB, R.getIterator());
if (Opcode == Instruction::Load) {
VPValue *OldLoad = R.getVPSingleValue();
VPValue *Load = Builder.createNaryOp(
>From b0d85ff48776c91988736041d4353a8600b2a08d Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 7 Sep 2026 16:33:45 +0000
Subject: [PATCH 09/21] Fixups
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 19d3b5ed7951c..56e5131c4b206 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1852,6 +1852,9 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::WidePtrAdd:
// WidePtrAdd supports scalar and vector base addresses.
return false;
+ case VPInstruction::VFMultipleLoad:
+ case VPInstruction::VFMultipleStore:
+ return Op == getOperand(0) || Op == getOperand(1) || Op == getOperand(2);
case VPInstruction::ExitingIVValue:
case VPInstruction::ExtractLane:
return Op == getOperand(0);
>From 1f635d281ec00e9d837ce173c6aa17c55ad330f5 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 11 Sep 2026 09:59:38 +0000
Subject: [PATCH 10/21] Rebase fixups
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 56e5131c4b206..ff798d012a89d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1194,8 +1194,7 @@ Value *VPInstruction::generate(VPTransformState &State,
}
case VPInstruction::VFMultipleLoad: {
unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
- auto *WideDataTy = VectorType::get(
- getScalarType(), State.VF.multiplyCoefficientBy(VFMultiple));
+ auto *WideDataTy = VectorType::get(getScalarType(), State.VF * VFMultiple);
Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
Align Alignment = Align(cast<VPConstantInt>(getOperand(2))->getZExtValue());
@@ -1205,8 +1204,7 @@ Value *VPInstruction::generate(VPTransformState &State,
case VPInstruction::VFMultipleStore: {
unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
Type *ScalarStoreTy = getOperand(3)->getScalarType();
- auto *WideDataTy = VectorType::get(
- ScalarStoreTy, State.VF.multiplyCoefficientBy(VFMultiple));
+ auto *WideDataTy = VectorType::get(ScalarStoreTy, State.VF * VFMultiple);
Value *WideData = PoisonValue::get(WideDataTy);
for (unsigned I = 0; I < VFMultiple; ++I) {
>From 847b1905864e9eae30881b15ede2fc19d2c2c660 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 11 Sep 2026 10:24:13 +0000
Subject: [PATCH 11/21] Fixups
---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 45 ++++++++++---------
1 file changed, 23 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f73392db568d2..f6fe65673ca26 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -328,29 +328,30 @@ void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
Store->addOperand(UnrollPart);
}
}
- } else {
- assert(VPI->getOpcode() == VPInstruction::VFMultipleLoad &&
- "Expected a load recipe");
- // We need to extract each unroll part as a subvector.
- auto ExtractPart0 = Builder.createNaryOp(
+ return;
+ }
+
+ assert(VPI->getOpcode() == VPInstruction::VFMultipleLoad &&
+ "Expected a VFMultipleLoad instruction");
+ // We need to extract each unroll part as a subvector.
+ auto *ExtractPart0 =
+ Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
+ {Groups[0]->getVPSingleValue(), getConstantInt(0)});
+ // First VPI with an extract of the first unroll part (ExtractPart0).
+ VPI->getVPSingleValue()->replaceUsesWithIf(
+ ExtractPart0, [&](VPUser &U, unsigned) { return &U != ExtractPart0; });
+ ToSkip.insert(ExtractPart0);
+
+ // Create extracts for the remaining unroll parts and remap later uses of
+ // ExtractPart0 to the correct unrolled part.
+ for (unsigned Part = 1; Part != UF; ++Part) {
+ VPInstruction *Group = Groups[Part / VFMultiple];
+ unsigned IndexInGroup = Part % VFMultiple;
+ auto *Extract = Builder.createNaryOp(
VPInstruction::ExtractVectorForPart,
- {Groups[0]->getVPSingleValue(), getConstantInt(0)});
- // First VPI with an extract of the first unroll part (ExtractPart0).
- VPI->getVPSingleValue()->replaceUsesWithIf(
- ExtractPart0, [&](VPUser &U, unsigned) { return &U != ExtractPart0; });
- ToSkip.insert(ExtractPart0);
-
- // Create extracts for the remaining unroll parts and remap later uses of
- // ExtractPart0 to the correct unrolled part.
- for (unsigned Part = 1; Part != UF; ++Part) {
- VPInstruction *Group = Groups[Part / VFMultiple];
- unsigned IndexInGroup = Part % VFMultiple;
- auto *Extract = Builder.createNaryOp(
- VPInstruction::ExtractVectorForPart,
- {Group->getVPSingleValue(), getConstantInt(IndexInGroup)});
- addRecipeForPart(ExtractPart0, Extract, Part);
- ToSkip.insert(Extract);
- }
+ {Group->getVPSingleValue(), getConstantInt(IndexInGroup)});
+ addRecipeForPart(ExtractPart0, Extract, Part);
+ ToSkip.insert(Extract);
}
}
>From 111344c3652b30772a51754b761c858d7e677891 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 15 Sep 2026 10:45:35 +0000
Subject: [PATCH 12/21] Fixups
---
.../llvm/Analysis/TargetTransformInfo.h | 9 +++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 5 ++
llvm/lib/Analysis/TargetTransformInfo.cpp | 6 +++
.../AArch64/AArch64TargetTransformInfo.cpp | 18 ++++---
.../AArch64/AArch64TargetTransformInfo.h | 3 ++
.../Transforms/Vectorize/LoopVectorize.cpp | 5 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 17 +++---
.../AArch64/multi-vector-mem-ops.ll | 54 +++++++++++++++++++
.../VPlan/vplan-print-before-after-all.ll | 1 -
11 files changed, 98 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index ed0be2c8ede7a..8dd96f3a575d5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1001,10 +1001,15 @@ class TargetTransformInfo {
unsigned Opcode1,
const SmallBitVector &OpcodeMask) const;
+ /// Returns the maximum VF multiple that can be used for a contiguous
+ /// load/store for the given \p VF and \p UF.
+ LLVM_ABI unsigned getMaximumVFMultipleForMemoryOp(ElementCount VF,
+ unsigned UF) const;
+
/// Return the preferred multiple of VF to use for a contiguous load/store.
/// Returning 1 leaves the operation at VF. The returned value must divide UF.
- /// \p CastHint is non-null if the stored or loaded valued is produced by or
- /// consumed a cast instruction respectively.
+ /// \p CastHint is non-null if the stored value is produced by a cast
+ /// instruction or the loaded value is consumed by one.
///
/// \p Opcode must be either Instruction::Load or Instruction::Store.
LLVM_ABI unsigned getPreferredVFMultipleForMemoryOp(
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index f70257b10fbac..57073b8d87a10 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -429,6 +429,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
+ virtual unsigned getMaximumVFMultipleForMemoryOp(ElementCount VF,
+ unsigned UF) const {
+ return 1;
+ }
+
virtual unsigned getPreferredVFMultipleForMemoryOp(
unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
bool IsMasked, std::optional<Instruction::CastOps> CastHint) const {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index a54c1fbdc5ffb..4ead8c70d26cb 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -554,6 +554,12 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
+unsigned
+TargetTransformInfo::getMaximumVFMultipleForMemoryOp(ElementCount VF,
+ unsigned UF) const {
+ return TTIImpl->getMaximumVFMultipleForMemoryOp(VF, UF);
+}
+
unsigned TargetTransformInfo::getPreferredVFMultipleForMemoryOp(
unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
bool IsMasked, std::optional<Instruction::CastOps> CastHint) const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 29b5719a09fb0..4d604ad390471 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5974,6 +5974,18 @@ bool AArch64TTIImpl::isLegalSpeculativeLoad(Type *DataType,
Size.getFixedValue() <= 16;
}
+unsigned AArch64TTIImpl::getMaximumVFMultipleForMemoryOp(ElementCount VF,
+ unsigned UF) const {
+ if (!ST->enableSubRegLiveness())
+ return 1;
+
+ if (!ST->hasSVE2p1() || !VF.isScalable() || !isPowerOf2_32(UF))
+ return 1;
+
+ // +sve2p1 multi-vector loads/stores can handle up to four vectors.
+ return std::min(4U, UF);
+}
+
unsigned AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(
unsigned Opcode, Type *DataTy, ElementCount VF, unsigned UF, bool IsMasked,
std::optional<Instruction::CastOps> CastHint) const {
@@ -5989,12 +6001,6 @@ unsigned AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(
(CastHint == Instruction::ZExt || CastHint == Instruction::SExt))
return 1;
- if (!ST->enableSubRegLiveness())
- return 1;
-
- if (!ST->hasSVE2p1() || !VF.isScalable() || !isPowerOf2_32(UF))
- return 1;
-
unsigned VectorWidth = VF.getKnownMinValue() * DL.getTypeSizeInBits(DataTy);
if (VectorWidth % 128 != 0)
return 1;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index aa1d059406c8c..042774c539a44 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -282,6 +282,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isLegalSpeculativeLoad(Type *DataType,
unsigned AddressSpace) const override;
+ unsigned getMaximumVFMultipleForMemoryOp(ElementCount VF,
+ unsigned UF) const override;
+
unsigned getPreferredVFMultipleForMemoryOp(
unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
bool IsMasked,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 201f023de7fa6..f99983372e212 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5792,8 +5792,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
*PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
- RUN_VPLAN_PASS(VPlanTransforms::scaleMemoryAccessesByUF, BestVPlan, BestVF,
- BestUF, TTI);
+ if (TTI.getMaximumVFMultipleForMemoryOp(BestVF, BestUF) > 1)
+ RUN_VPLAN_PASS(VPlanTransforms::scaleMemoryAccessesByUF, BestVPlan, BestVF,
+ BestUF, TTI);
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ff798d012a89d..947f1a20e648f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1838,6 +1838,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::Intrinsic:
case VPInstruction::ReductionStartVector:
case VPInstruction::ResumeForEpilogue:
+ case VPInstruction::VFMultipleLoad:
return true;
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
@@ -1850,7 +1851,6 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::WidePtrAdd:
// WidePtrAdd supports scalar and vector base addresses.
return false;
- case VPInstruction::VFMultipleLoad:
case VPInstruction::VFMultipleStore:
return Op == getOperand(0) || Op == getOperand(1) || Op == getOperand(2);
case VPInstruction::ExitingIVValue:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d87e623892ced..00eace3bc7d7d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4199,11 +4199,9 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
unsigned UF,
const TargetTransformInfo &TTI) {
- if (UF == 1)
- return;
+ assert(UF > 1 && "Expected plan to have an UF > 1");
Type *IVTy = Plan.getVectorLoopRegion()->getCanonicalIVType();
-
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f6fe65673ca26..d442d407938bd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -294,8 +294,9 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
}
void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
- assert(VPI->getOpcode() == VPInstruction::VFMultipleLoad ||
- VPI->getOpcode() == VPInstruction::VFMultipleStore);
+ assert((VPI->getOpcode() == VPInstruction::VFMultipleLoad ||
+ VPI->getOpcode() == VPInstruction::VFMultipleStore) &&
+ "expected a vf-multiple load/store");
unsigned VFMultiple = cast<VPConstantInt>(VPI->getOperand(0))->getZExtValue();
assert(VFMultiple > 1 && UF % VFMultiple == 0 &&
@@ -331,16 +332,12 @@ void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
return;
}
- assert(VPI->getOpcode() == VPInstruction::VFMultipleLoad &&
- "Expected a VFMultipleLoad instruction");
// We need to extract each unroll part as a subvector.
- auto *ExtractPart0 =
- Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
- {Groups[0]->getVPSingleValue(), getConstantInt(0)});
+ auto *ExtractPart0 = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
+ {Groups[0], getConstantInt(0)});
// First VPI with an extract of the first unroll part (ExtractPart0).
- VPI->getVPSingleValue()->replaceUsesWithIf(
+ VPI->replaceUsesWithIf(
ExtractPart0, [&](VPUser &U, unsigned) { return &U != ExtractPart0; });
- ToSkip.insert(ExtractPart0);
// Create extracts for the remaining unroll parts and remap later uses of
// ExtractPart0 to the correct unrolled part.
@@ -351,7 +348,6 @@ void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
VPInstruction::ExtractVectorForPart,
{Group->getVPSingleValue(), getConstantInt(IndexInGroup)});
addRecipeForPart(ExtractPart0, Extract, Part);
- ToSkip.insert(Extract);
}
}
@@ -372,7 +368,6 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
return;
}
}
-
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
if (isa<StoreInst>(RepR->getUnderlyingValue()) &&
RepR->getOperand(1)->isDefinedOutsideLoopRegions()) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index b90e29519e24e..398e2e0613e25 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -603,3 +603,57 @@ for.body:
exit:
ret void
}
+
+define void @store_address(ptr noalias %x, ptr noalias %y, i64 %n) {
+; UNMASKED-SVE2P1-LABEL: define void @store_address(
+; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
+; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_PH]]:
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 1
+; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
+; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_2:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
+; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_3:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[VEC_IND]]
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x ptr> [[WIDE_GEP]], i64 0
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_2]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> poison, <vscale x 2 x ptr> [[WIDE_GEP]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP6]], <vscale x 2 x ptr> [[WIDE_GEP1]], i64 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP7]], <vscale x 2 x ptr> [[WIDE_GEP2]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP8]], <vscale x 2 x ptr> [[WIDE_GEP3]], i64 6)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x ptr> [[TMP9]], ptr [[TMP5]], align 8
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %px = getelementptr inbounds i64, ptr %x, i64 %iv
+ store ptr %px, ptr %px, align 8
+ %next = add nuw i64 %iv, 1
+ %cmp = icmp ult i64 %next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index cb492acc29e1e..5fd844e186e44 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -71,7 +71,6 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printOptimizedVPlan
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMinimumIterationCheck
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceWideCanonicalIVWithWideIV
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::scaleMemoryAccessesByUF
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::unrollByUF
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializePacksAndUnpacks
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeBroadcasts
>From d185feff22515f2be467e960ddb0be0333128ffb Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 28 Sep 2026 15:16:30 +0000
Subject: [PATCH 13/21] Fixups
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 28 ++++++++-----------
.../Transforms/Vectorize/VPlanTransforms.h | 5 ++--
...-vector-mem-ops-non-power-of-two-unroll.ll | 2 +-
.../vplan-printing-multi-vector-mem-ops.ll | 4 +--
5 files changed, 20 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f99983372e212..5164c6de2fc8d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5793,8 +5793,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
if (TTI.getMaximumVFMultipleForMemoryOp(BestVF, BestUF) > 1)
- RUN_VPLAN_PASS(VPlanTransforms::scaleMemoryAccessesByUF, BestVPlan, BestVF,
- BestUF, TTI);
+ RUN_VPLAN_PASS(VPlanTransforms::widenMemoryAccessesToVFMultiple, BestVPlan,
+ BestVF, BestUF, TTI);
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 00eace3bc7d7d..ffd41692dd1b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4196,23 +4196,19 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
}
}
-void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
- unsigned UF,
- const TargetTransformInfo &TTI) {
+void VPlanTransforms::widenMemoryAccessesToVFMultiple(
+ VPlan &Plan, ElementCount VF, unsigned UF, const TargetTransformInfo &TTI) {
assert(UF > 1 && "Expected plan to have an UF > 1");
- Type *IVTy = Plan.getVectorLoopRegion()->getCanonicalIVType();
+ Type *I64Ty = IntegerType::getInt64Ty(Plan.getContext());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- uint64_t Stride;
VPValue *StoredValue = nullptr;
- auto m_ConstantStrideVecPtr =
- m_VecPtr(m_VPValue(), m_ConstantInt(Stride));
- if ((!match(&R, m_WidenLoad(m_ConstantStrideVecPtr)) &&
- !match(&R, m_WidenStore(m_ConstantStrideVecPtr,
- m_VPValue(StoredValue)))) ||
- Stride != 1)
+ auto m_ContiguousVecPtr = m_VecPtr(m_VPValue(), m_One());
+ if ((!match(&R, m_WidenLoad(m_ContiguousVecPtr)) &&
+ !match(&R,
+ m_WidenStore(m_ContiguousVecPtr, m_VPValue(StoredValue)))))
continue;
auto *MemOp = cast<VPWidenMemoryRecipe>(&R);
@@ -4244,21 +4240,21 @@ void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
continue;
VPValue *Ptr = MemOp->getAddr();
- VPValue *VFMultipleVPV = Plan.getConstantInt(IVTy, VFMultiple);
- VPValue *Align = Plan.getConstantInt(IVTy, MemOp->getAlign().value());
+ VPValue *VFMultipleVPV = Plan.getConstantInt(I64Ty, VFMultiple);
+ VPValue *Align = Plan.getConstantInt(I64Ty, MemOp->getAlign().value());
VPBuilder Builder(VPBB, R.getIterator());
if (Opcode == Instruction::Load) {
VPValue *OldLoad = R.getVPSingleValue();
VPValue *Load = Builder.createNaryOp(
VPInstruction::VFMultipleLoad, {VFMultipleVPV, Ptr, Align}, nullptr,
- {}, {}, DebugLoc::getUnknown(), "", OldLoad->getScalarType());
+ {}, *MemOp, R.getDebugLoc(), "", OldLoad->getScalarType());
OldLoad->replaceAllUsesWith(Load);
} else {
assert(Opcode == Instruction::Store);
Builder.createNaryOp(VPInstruction::VFMultipleStore,
- {VFMultipleVPV, Ptr, Align, StoredValue},
- R.getDebugLoc());
+ {VFMultipleVPV, Ptr, Align, StoredValue}, nullptr,
+ {}, *MemOp, R.getDebugLoc());
}
R.eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index f59beb0e6b1e9..d4f246b13e2a2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -484,8 +484,9 @@ struct VPlanTransforms {
/// Widens memory operations by a factor of UF based on a target hook.
/// This allows targets to use wider memory operations when profitable.
- static void scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF, unsigned UF,
- const TargetTransformInfo &TTI);
+ static void widenMemoryAccessesToVFMultiple(VPlan &Plan, ElementCount VF,
+ unsigned UF,
+ const TargetTransformInfo &TTI);
// Materialize vector trip counts for constants early if it can simply be
// computed as (Original TC / VF * UF) * VF * UF.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
index ac9e55117364e..529909d353e17 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
@@ -3,7 +3,7 @@
target triple = "aarch64-unknown-linux-gnu"
-; Tests `scaleMemoryAccessesByUF` with a non-power-of-two unroll factor.
+; Tests `widenMemoryAccessesToVFMultiple` with a non-power-of-two unroll factor.
; On AArch64, this should be rejected (which results in `vscale x 4` loads/stores).
define void @mixed_i64_i32_accesses(ptr noalias %x, ptr noalias %y, i64 %n) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
index 15dc08babf853..82b1201c22629 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
-; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-before="scaleMemoryAccessesByUF$" 2>&1 \
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-before="widenMemoryAccessesToVFMultiple$" 2>&1 \
; RUN: | FileCheck --check-prefix=BEFORE-SCALE %s
-; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="scaleMemoryAccessesByUF$" 2>&1 \
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="widenMemoryAccessesToVFMultiple$" 2>&1 \
; RUN: | FileCheck --check-prefix=AFTER-SCALE %s
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="unrollByUF$" 2>&1 \
; RUN: | FileCheck --check-prefix=AFTER-UNROLL %s
>From 07fb066a9f8e0d7b9987a83eac83c299b37b9782 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 28 Sep 2026 15:39:43 +0000
Subject: [PATCH 14/21] Fixups
---
.../AArch64/multi-vector-mem-ops.ll | 93 +++++++++++--------
1 file changed, 53 insertions(+), 40 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index 398e2e0613e25..3b41923deada9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -58,7 +58,7 @@ define void @mixed_i64_i32_accesses(ptr noalias %x, ptr noalias %y, i64 %n) {
; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP30]], ptr [[TMP18]], align 4
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -152,7 +152,7 @@ define void @mixed_i32_more_frequent_than_i64(ptr noalias %x, ptr noalias %y, pt
; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP43]], ptr [[TMP31]], align 4
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -219,7 +219,7 @@ define void @first_order_recurrence_i64_scaled_load_and_store(ptr noalias %src,
; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP12]], ptr [[TMP16]], align 8
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -273,7 +273,7 @@ define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP7]]
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -329,10 +329,10 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = freeze <vscale x 2 x i1> [[TMP11]]
; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = freeze <vscale x 2 x i1> [[TMP12]]
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = or <vscale x 2 x i1> [[TMP15]], [[TMP16]]
; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = freeze <vscale x 2 x i1> [[TMP13]]
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = or <vscale x 2 x i1> [[TMP17]], [[TMP18]]
; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = freeze <vscale x 2 x i1> [[TMP14]]
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = or <vscale x 2 x i1> [[TMP15]], [[TMP16]]
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = or <vscale x 2 x i1> [[TMP32]], [[TMP18]]
; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = or <vscale x 2 x i1> [[TMP19]], [[TMP20]]
; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP21]])
; UNMASKED-SVE2P1-NEXT: [[TMP23]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP2]]
@@ -345,7 +345,7 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: [[TMP30]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI3]]
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -414,7 +414,7 @@ define void @extending_load(ptr noalias %dst, ptr %src, i64 %n) {
; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP22]], ptr [[TMP18]], align 8
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -500,7 +500,7 @@ define void @reverse_stride(ptr noalias %c, ptr noalias %a, ptr noalias %b, i64
; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP24]], ptr [[TMP32]], align 4
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -544,16 +544,13 @@ define void @gather_nxv4i32_stride2(ptr noalias %a, ptr noalias %b, i64 %n) {
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = add i64 [[TMP2]], 0
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = mul i64 [[TMP2]], 1
; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = shl i64 [[TMP2]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = mul i64 [[TMP8]], 1
; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = mul i64 [[TMP2]], 3
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 0
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 1
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = mul i64 [[TMP12]], 1
; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 1
; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = shl i64 [[TMP7]], 1
@@ -583,7 +580,7 @@ define void @gather_nxv4i32_stride2(ptr noalias %a, ptr noalias %b, i64 %n) {
; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x float> [[TMP32]], ptr [[TMP28]], align 4
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
@@ -622,38 +619,54 @@ define void @store_address(ptr noalias %x, ptr noalias %y, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_2:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
-; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_3:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[VEC_IND]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG26:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG26]]
+; UNMASKED-SVE2P1-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
+; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_2:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
+; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_3:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[VEC_IND]], !dbg [[DBG30:![0-9]+]]
; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x ptr> [[WIDE_GEP]], i64 0
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_2]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_3]]
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> poison, <vscale x 2 x ptr> [[WIDE_GEP]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP6]], <vscale x 2 x ptr> [[WIDE_GEP1]], i64 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP7]], <vscale x 2 x ptr> [[WIDE_GEP2]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP8]], <vscale x 2 x ptr> [[WIDE_GEP3]], i64 6)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x ptr> [[TMP9]], ptr [[TMP5]], align 8
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD]], !dbg [[DBG30]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_2]], !dbg [[DBG30]]
+; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_3]], !dbg [[DBG30]]
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> poison, <vscale x 2 x ptr> [[WIDE_GEP]], i64 0), !dbg [[DBG31:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP6]], <vscale x 2 x ptr> [[WIDE_GEP1]], i64 2), !dbg [[DBG31]]
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP7]], <vscale x 2 x ptr> [[WIDE_GEP2]], i64 4), !dbg [[DBG31]]
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP8]], <vscale x 2 x ptr> [[WIDE_GEP3]], i64 6), !dbg [[DBG31]]
+; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x ptr> [[TMP9]], ptr [[TMP5]], align 8, !dbg [[DBG31]]
+; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]], !dbg [[DBG26]]
+; UNMASKED-SVE2P1-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG32:![0-9]+]]
+; UNMASKED-SVE2P1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG32]], !llvm.loop [[LOOP33:![0-9]+]]
; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
- %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
- %px = getelementptr inbounds i64, ptr %x, i64 %iv
- store ptr %px, ptr %px, align 8
- %next = add nuw i64 %iv, 1
- %cmp = icmp ult i64 %next, %n
- br i1 %cmp, label %loop, label %exit
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ], !dbg !3
+ %px = getelementptr inbounds i64, ptr %x, i64 %iv, !dbg !4
+ store ptr %px, ptr %px, align 8, !dbg !5
+ %next = add nuw i64 %iv, 1, !dbg !6
+ %cmp = icmp ult i64 %next, %n, !dbg !7
+ br i1 %cmp, label %loop, label %exit, !dbg !8
exit:
ret void
}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 24.0.0git")
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !DILocation(line: 4, scope: !9)
+!4 = !DILocation(line: 5, scope: !9)
+!5 = !DILocation(line: 6, scope: !9)
+!6 = !DILocation(line: 7, scope: !9)
+!7 = !DILocation(line: 8, scope: !9)
+!8 = !DILocation(line: 9, scope: !9)
+!9 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 11, type: !10, unit: !0, retainedNodes: !11)
+!10 = distinct !DISubroutineType(types: !11)
+!11 = !{}
>From 1725652524120fe30821e303c2e169a284dc5287 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 30 Sep 2026 16:48:48 +0000
Subject: [PATCH 15/21] Tweak opcodes
---
llvm/lib/Transforms/Vectorize/VPlan.h | 14 ++++++--------
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 11 ++++++-----
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 12 +++++++-----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 13 ++++++++++---
.../AArch64/vplan-printing-multi-vector-mem-ops.ll | 6 +++---
5 files changed, 32 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index dcf15a8c3102d..60b1f870209ca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1317,15 +1317,13 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// WideActiveLaneMask is used for control flow and is unrolled by widening,
// with one extract vector created per unroll part.
WideActiveLaneMask,
- // Signature: (VFMultiple, Address, Alignment) -> Wide Vector
- // Loads a single wide vector of `VFMultiple * VF` elements. VFMultiple must
- // divide UF. After unrolling, each section of VF elements in the wide
- // vector corresponds to an unroll part.
+ // (PreferredVFMultiple, Address, Alignment, VFMultiple) -> Wide Vector
+ // Loads a single wide vector of `VFMultiple * VF` elements. The
+ // `PreferredVFMultiple` is used as a hint to guide unrolling.
VFMultipleLoad,
- // Signature: (VFMultiple, Address, Alignment, Vectors...)
- // Concatenates VFMultiple vector operands into a single wide vector of
- // `VFMultiple * VF` elements and stores it. After unrolling, each vector
- // operand corresponds to an unroll part.
+ // (PreferredVFMultiple, Address, Alignment, Vectors...)
+ // Concatenates all vector operands into a single wide vector and stores the
+ // result. The `PreferredVFMultiple` is used as a hint to guide unrolling.
VFMultipleStore,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 947f1a20e648f..d441fa61f1960 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -682,8 +682,9 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::Select:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
- case VPInstruction::VFMultipleLoad:
return 3;
+ case VPInstruction::VFMultipleLoad:
+ return 4;
case Instruction::Call:
return getCalledFnOperandIndex(operands()) + 1;
case Instruction::GetElementPtr:
@@ -1193,7 +1194,7 @@ Value *VPInstruction::generate(VPTransformState &State,
/*FMFSource=*/nullptr, getName());
}
case VPInstruction::VFMultipleLoad: {
- unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
+ unsigned VFMultiple = cast<VPConstantInt>(getOperand(3))->getZExtValue();
auto *WideDataTy = VectorType::get(getScalarType(), State.VF * VFMultiple);
Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
@@ -1202,12 +1203,12 @@ Value *VPInstruction::generate(VPTransformState &State,
"vf.multiple.load");
}
case VPInstruction::VFMultipleStore: {
- unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
+ unsigned NumVectorOps = getNumOperands() - 3;
Type *ScalarStoreTy = getOperand(3)->getScalarType();
- auto *WideDataTy = VectorType::get(ScalarStoreTy, State.VF * VFMultiple);
+ auto *WideDataTy = VectorType::get(ScalarStoreTy, State.VF * NumVectorOps);
Value *WideData = PoisonValue::get(WideDataTy);
- for (unsigned I = 0; I < VFMultiple; ++I) {
+ for (unsigned I = 0; I < NumVectorOps; ++I) {
Value *Part = State.get(getOperand(I + 3));
WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part,
I * State.VF.getKnownMinValue());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ffd41692dd1b6..a5ab632895f6b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4240,21 +4240,23 @@ void VPlanTransforms::widenMemoryAccessesToVFMultiple(
continue;
VPValue *Ptr = MemOp->getAddr();
- VPValue *VFMultipleVPV = Plan.getConstantInt(I64Ty, VFMultiple);
+ VPValue *PreferredVFMultipleVPV = Plan.getConstantInt(I64Ty, VFMultiple);
VPValue *Align = Plan.getConstantInt(I64Ty, MemOp->getAlign().value());
VPBuilder Builder(VPBB, R.getIterator());
if (Opcode == Instruction::Load) {
VPValue *OldLoad = R.getVPSingleValue();
VPValue *Load = Builder.createNaryOp(
- VPInstruction::VFMultipleLoad, {VFMultipleVPV, Ptr, Align}, nullptr,
- {}, *MemOp, R.getDebugLoc(), "", OldLoad->getScalarType());
+ VPInstruction::VFMultipleLoad,
+ {PreferredVFMultipleVPV, Ptr, Align,
+ /*VFMultiple=*/Plan.getConstantInt(I64Ty, 1)},
+ nullptr, {}, *MemOp, R.getDebugLoc(), "", OldLoad->getScalarType());
OldLoad->replaceAllUsesWith(Load);
} else {
assert(Opcode == Instruction::Store);
Builder.createNaryOp(VPInstruction::VFMultipleStore,
- {VFMultipleVPV, Ptr, Align, StoredValue}, nullptr,
- {}, *MemOp, R.getDebugLoc());
+ {PreferredVFMultipleVPV, Ptr, Align, StoredValue},
+ nullptr, {}, *MemOp, R.getDebugLoc());
}
R.eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index d442d407938bd..a0bb627b9414b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -298,10 +298,13 @@ void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
VPI->getOpcode() == VPInstruction::VFMultipleStore) &&
"expected a vf-multiple load/store");
- unsigned VFMultiple = cast<VPConstantInt>(VPI->getOperand(0))->getZExtValue();
- assert(VFMultiple > 1 && UF % VFMultiple == 0 &&
- "expected VFMultiple to divide UF");
+ unsigned PreferredVFMultiple =
+ cast<VPConstantInt>(VPI->getOperand(0))->getZExtValue();
+ assert(PreferredVFMultiple > 1 && UF % PreferredVFMultiple == 0 &&
+ "expected PreferredVFMultiple to divide UF");
+ // For now simply follow the preferred VFMultiple.
+ unsigned VFMultiple = PreferredVFMultiple;
SmallVector<VPInstruction *, 4> Groups(UF / VFMultiple, nullptr);
Groups[0] = VPI;
@@ -332,6 +335,10 @@ void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
return;
}
+ // Set the VFMultiple to match the PreferredVFMultiple.
+ for (VPInstruction *Load : Groups)
+ Load->setOperand(3, VPI->getOperand(0));
+
// We need to extract each unroll part as a subvector.
auto *ExtractPart0 = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
{Groups[0], getConstantInt(0)});
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
index 82b1201c22629..c56f9e7bc7277 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
@@ -69,7 +69,7 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-SCALE-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; AFTER-SCALE-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
; AFTER-SCALE-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-SCALE-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP9]]>, ir<8>
+; AFTER-SCALE-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP9]]>, ir<8>, ir<1>
; AFTER-SCALE-NEXT: WIDEN ir<%add> = add vp<[[VP10]]>, ir<1>
; AFTER-SCALE-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
; AFTER-SCALE-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP11]]>, ir<8>, ir<%add>
@@ -108,8 +108,8 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-UNROLL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
; AFTER-UNROLL-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
; AFTER-UNROLL-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP9]]>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP10]]>, ir<8>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP11]]>, ir<8>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP10]]>, ir<8>, ir<2>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP11]]>, ir<8>, ir<2>
; AFTER-UNROLL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<0>
; AFTER-UNROLL-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<1>
; AFTER-UNROLL-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-vector-for-part vp<[[VP13]]>, ir<0>
>From 68db49beba4685468339512a8e251c3638658356 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 30 Sep 2026 17:45:38 +0000
Subject: [PATCH 16/21] Revert "Tweak opcodes"
This reverts commit cad40b17be4157e28b6edb5c6258d5eb8eb3049e.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 14 ++++++++------
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 11 +++++------
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 12 +++++-------
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 13 +++----------
.../AArch64/vplan-printing-multi-vector-mem-ops.ll | 6 +++---
5 files changed, 24 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 60b1f870209ca..dcf15a8c3102d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1317,13 +1317,15 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// WideActiveLaneMask is used for control flow and is unrolled by widening,
// with one extract vector created per unroll part.
WideActiveLaneMask,
- // (PreferredVFMultiple, Address, Alignment, VFMultiple) -> Wide Vector
- // Loads a single wide vector of `VFMultiple * VF` elements. The
- // `PreferredVFMultiple` is used as a hint to guide unrolling.
+ // Signature: (VFMultiple, Address, Alignment) -> Wide Vector
+ // Loads a single wide vector of `VFMultiple * VF` elements. VFMultiple must
+ // divide UF. After unrolling, each section of VF elements in the wide
+ // vector corresponds to an unroll part.
VFMultipleLoad,
- // (PreferredVFMultiple, Address, Alignment, Vectors...)
- // Concatenates all vector operands into a single wide vector and stores the
- // result. The `PreferredVFMultiple` is used as a hint to guide unrolling.
+ // Signature: (VFMultiple, Address, Alignment, Vectors...)
+ // Concatenates VFMultiple vector operands into a single wide vector of
+ // `VFMultiple * VF` elements and stores it. After unrolling, each vector
+ // operand corresponds to an unroll part.
VFMultipleStore,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d441fa61f1960..947f1a20e648f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -682,9 +682,8 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::Select:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
- return 3;
case VPInstruction::VFMultipleLoad:
- return 4;
+ return 3;
case Instruction::Call:
return getCalledFnOperandIndex(operands()) + 1;
case Instruction::GetElementPtr:
@@ -1194,7 +1193,7 @@ Value *VPInstruction::generate(VPTransformState &State,
/*FMFSource=*/nullptr, getName());
}
case VPInstruction::VFMultipleLoad: {
- unsigned VFMultiple = cast<VPConstantInt>(getOperand(3))->getZExtValue();
+ unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
auto *WideDataTy = VectorType::get(getScalarType(), State.VF * VFMultiple);
Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
@@ -1203,12 +1202,12 @@ Value *VPInstruction::generate(VPTransformState &State,
"vf.multiple.load");
}
case VPInstruction::VFMultipleStore: {
- unsigned NumVectorOps = getNumOperands() - 3;
+ unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
Type *ScalarStoreTy = getOperand(3)->getScalarType();
- auto *WideDataTy = VectorType::get(ScalarStoreTy, State.VF * NumVectorOps);
+ auto *WideDataTy = VectorType::get(ScalarStoreTy, State.VF * VFMultiple);
Value *WideData = PoisonValue::get(WideDataTy);
- for (unsigned I = 0; I < NumVectorOps; ++I) {
+ for (unsigned I = 0; I < VFMultiple; ++I) {
Value *Part = State.get(getOperand(I + 3));
WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part,
I * State.VF.getKnownMinValue());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a5ab632895f6b..ffd41692dd1b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4240,23 +4240,21 @@ void VPlanTransforms::widenMemoryAccessesToVFMultiple(
continue;
VPValue *Ptr = MemOp->getAddr();
- VPValue *PreferredVFMultipleVPV = Plan.getConstantInt(I64Ty, VFMultiple);
+ VPValue *VFMultipleVPV = Plan.getConstantInt(I64Ty, VFMultiple);
VPValue *Align = Plan.getConstantInt(I64Ty, MemOp->getAlign().value());
VPBuilder Builder(VPBB, R.getIterator());
if (Opcode == Instruction::Load) {
VPValue *OldLoad = R.getVPSingleValue();
VPValue *Load = Builder.createNaryOp(
- VPInstruction::VFMultipleLoad,
- {PreferredVFMultipleVPV, Ptr, Align,
- /*VFMultiple=*/Plan.getConstantInt(I64Ty, 1)},
- nullptr, {}, *MemOp, R.getDebugLoc(), "", OldLoad->getScalarType());
+ VPInstruction::VFMultipleLoad, {VFMultipleVPV, Ptr, Align}, nullptr,
+ {}, *MemOp, R.getDebugLoc(), "", OldLoad->getScalarType());
OldLoad->replaceAllUsesWith(Load);
} else {
assert(Opcode == Instruction::Store);
Builder.createNaryOp(VPInstruction::VFMultipleStore,
- {PreferredVFMultipleVPV, Ptr, Align, StoredValue},
- nullptr, {}, *MemOp, R.getDebugLoc());
+ {VFMultipleVPV, Ptr, Align, StoredValue}, nullptr,
+ {}, *MemOp, R.getDebugLoc());
}
R.eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index a0bb627b9414b..d442d407938bd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -298,13 +298,10 @@ void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
VPI->getOpcode() == VPInstruction::VFMultipleStore) &&
"expected a vf-multiple load/store");
- unsigned PreferredVFMultiple =
- cast<VPConstantInt>(VPI->getOperand(0))->getZExtValue();
- assert(PreferredVFMultiple > 1 && UF % PreferredVFMultiple == 0 &&
- "expected PreferredVFMultiple to divide UF");
+ unsigned VFMultiple = cast<VPConstantInt>(VPI->getOperand(0))->getZExtValue();
+ assert(VFMultiple > 1 && UF % VFMultiple == 0 &&
+ "expected VFMultiple to divide UF");
- // For now simply follow the preferred VFMultiple.
- unsigned VFMultiple = PreferredVFMultiple;
SmallVector<VPInstruction *, 4> Groups(UF / VFMultiple, nullptr);
Groups[0] = VPI;
@@ -335,10 +332,6 @@ void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
return;
}
- // Set the VFMultiple to match the PreferredVFMultiple.
- for (VPInstruction *Load : Groups)
- Load->setOperand(3, VPI->getOperand(0));
-
// We need to extract each unroll part as a subvector.
auto *ExtractPart0 = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
{Groups[0], getConstantInt(0)});
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
index c56f9e7bc7277..82b1201c22629 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
@@ -69,7 +69,7 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-SCALE-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; AFTER-SCALE-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
; AFTER-SCALE-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-SCALE-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP9]]>, ir<8>, ir<1>
+; AFTER-SCALE-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP9]]>, ir<8>
; AFTER-SCALE-NEXT: WIDEN ir<%add> = add vp<[[VP10]]>, ir<1>
; AFTER-SCALE-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
; AFTER-SCALE-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP11]]>, ir<8>, ir<%add>
@@ -108,8 +108,8 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-UNROLL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
; AFTER-UNROLL-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
; AFTER-UNROLL-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP9]]>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP10]]>, ir<8>, ir<2>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP11]]>, ir<8>, ir<2>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP10]]>, ir<8>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP11]]>, ir<8>
; AFTER-UNROLL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<0>
; AFTER-UNROLL-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<1>
; AFTER-UNROLL-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-vector-for-part vp<[[VP13]]>, ir<0>
>From fbd938d51934ad44c57b84526c69ab57f3e29437 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 1 Oct 2026 10:07:02 +0000
Subject: [PATCH 17/21] Simplify operations
---
.../llvm/Analysis/TargetTransformInfo.h | 31 ++--
.../llvm/Analysis/TargetTransformInfoImpl.h | 14 +-
llvm/lib/Analysis/TargetTransformInfo.cpp | 16 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 52 +++----
.../AArch64/AArch64TargetTransformInfo.h | 9 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 21 ++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 76 +++++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 57 ++++---
.../Transforms/Vectorize/VPlanTransforms.h | 5 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 76 ++--------
...-vector-mem-ops-non-power-of-two-unroll.ll | 2 +-
.../AArch64/multi-vector-mem-ops.ll | 142 ++++++++----------
.../vplan-printing-multi-vector-mem-ops.ll | 45 +++---
14 files changed, 232 insertions(+), 320 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 8dd96f3a575d5..91750f5ec5652 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1001,20 +1001,25 @@ class TargetTransformInfo {
unsigned Opcode1,
const SmallBitVector &OpcodeMask) const;
- /// Returns the maximum VF multiple that can be used for a contiguous
- /// load/store for the given \p VF and \p UF.
- LLVM_ABI unsigned getMaximumVFMultipleForMemoryOp(ElementCount VF,
- unsigned UF) const;
-
- /// Return the preferred multiple of VF to use for a contiguous load/store.
- /// Returning 1 leaves the operation at VF. The returned value must divide UF.
- /// \p CastHint is non-null if the stored value is produced by a cast
- /// instruction or the loaded value is consumed by one.
+ enum MaskSource {
+ /// The operation is unmasked.
+ MS_None,
+ /// The operation is masked with an arbitrary predicate.
+ MS_Masked,
+ /// The operation is masked with a contiguous active lane mask.
+ MS_ActiveLaneMask,
+ };
+
+ /// Return true if the target supports loading or storing \p NumVectors
+ /// contiguous vectors of type \p VectorTy as a single operation. A null
+ /// \p VectorTy queries whether the target supports this operation in
+ /// general.
///
- /// \p Opcode must be either Instruction::Load or Instruction::Store.
- LLVM_ABI unsigned getPreferredVFMultipleForMemoryOp(
- unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
- bool IsMasked = false,
+ /// \p CastHint is non-null if a stored value is produced by a cast or a
+ /// loaded value is consumed by one.
+ LLVM_ABI bool hasMultipleVectorLoadStore(
+ unsigned NumVectors, VectorType *VectorTy = nullptr, bool IsStore = false,
+ MaskSource Mask = MS_None,
std::optional<Instruction::CastOps> CastHint = std::nullopt) const;
/// Return true if we should be enabling ordered reductions for the target.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 57073b8d87a10..cd4acc0731c7d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -429,15 +429,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
- virtual unsigned getMaximumVFMultipleForMemoryOp(ElementCount VF,
- unsigned UF) const {
- return 1;
- }
-
- virtual unsigned getPreferredVFMultipleForMemoryOp(
- unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
- bool IsMasked, std::optional<Instruction::CastOps> CastHint) const {
- return 1;
+ virtual bool hasMultipleVectorLoadStore(
+ unsigned NumVectors, VectorType *VectorTy, bool IsStore,
+ TTI::MaskSource Mask,
+ std::optional<Instruction::CastOps> CastHint) const {
+ return false;
}
virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 4ead8c70d26cb..8e3bb78b71218 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -554,17 +554,11 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
-unsigned
-TargetTransformInfo::getMaximumVFMultipleForMemoryOp(ElementCount VF,
- unsigned UF) const {
- return TTIImpl->getMaximumVFMultipleForMemoryOp(VF, UF);
-}
-
-unsigned TargetTransformInfo::getPreferredVFMultipleForMemoryOp(
- unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
- bool IsMasked, std::optional<Instruction::CastOps> CastHint) const {
- return TTIImpl->getPreferredVFMultipleForMemoryOp(Opcode, DataType, VF, UF,
- IsMasked, CastHint);
+bool TargetTransformInfo::hasMultipleVectorLoadStore(
+ unsigned NumVectors, VectorType *VectorTy, bool IsStore,
+ TTI::MaskSource Mask, std::optional<Instruction::CastOps> CastHint) const {
+ return TTIImpl->hasMultipleVectorLoadStore(NumVectors, VectorTy, IsStore,
+ Mask, CastHint);
}
bool TargetTransformInfo::isLegalInterleavedAccessType(
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 4d604ad390471..8954aab4cb781 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5974,46 +5974,36 @@ bool AArch64TTIImpl::isLegalSpeculativeLoad(Type *DataType,
Size.getFixedValue() <= 16;
}
-unsigned AArch64TTIImpl::getMaximumVFMultipleForMemoryOp(ElementCount VF,
- unsigned UF) const {
- if (!ST->enableSubRegLiveness())
- return 1;
+bool AArch64TTIImpl::hasMultipleVectorLoadStore(
+ unsigned NumVectors, VectorType *VectorTy, bool IsStore,
+ TTI::MaskSource Mask, std::optional<Instruction::CastOps> CastHint) const {
+ if (NumVectors <= 1 || !ST->enableSubRegLiveness() || !ST->hasSVE2p1())
+ return false;
- if (!ST->hasSVE2p1() || !VF.isScalable() || !isPowerOf2_32(UF))
- return 1;
+ // TODO: Support masked multi-vector loads/stores.
+ if (Mask != TTI::MS_None)
+ return false;
- // +sve2p1 multi-vector loads/stores can handle up to four vectors.
- return std::min(4U, UF);
-}
+ // A null vector type queries whether the target supports multi-vector memory
+ // operations in general.
+ if (!VectorTy)
+ return true;
-unsigned AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(
- unsigned Opcode, Type *DataTy, ElementCount VF, unsigned UF, bool IsMasked,
- std::optional<Instruction::CastOps> CastHint) const {
- assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
- "expected load/store opcode");
- if (IsMasked)
- return 1; // TODO: Support masked multi-vector loads/stores.
+ if (!isa<ScalableVectorType>(VectorTy))
+ return false;
// Conservatively, avoid using multi-vector loads when it's possible we could
// use extending loads instead. Note: We can ignore stores as we only use
// truncating stores when the store vector-width is < a full SVE vector.
- if (Opcode == Instruction::Load &&
+ if (!IsStore &&
(CastHint == Instruction::ZExt || CastHint == Instruction::SExt))
- return 1;
-
- unsigned VectorWidth = VF.getKnownMinValue() * DL.getTypeSizeInBits(DataTy);
- if (VectorWidth % 128 != 0)
- return 1;
-
- for (unsigned TargetWidth : {512u, 256u}) {
- if (TargetWidth % VectorWidth == 0) {
- unsigned Scale = TargetWidth / VectorWidth;
- if (Scale <= UF)
- return Scale;
- }
- }
+ return false;
- return 1;
+ // For unpredicated loads/stores allow any pow-of-two multiple of a vector >=
+ // to a single z-register. We can split operations wider than a single
+ // multi-vector load/store during ISEL.
+ unsigned VectorWidth = DL.getTypeSizeInBits(VectorTy).getKnownMinValue();
+ return VectorWidth % 128 == 0 && isPowerOf2_32(NumVectors);
}
unsigned
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 042774c539a44..010b1f9142edd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -282,12 +282,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isLegalSpeculativeLoad(Type *DataType,
unsigned AddressSpace) const override;
- unsigned getMaximumVFMultipleForMemoryOp(ElementCount VF,
- unsigned UF) const override;
-
- unsigned getPreferredVFMultipleForMemoryOp(
- unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
- bool IsMasked,
+ bool hasMultipleVectorLoadStore(
+ unsigned NumVectors, VectorType *VectorTy, bool IsStore,
+ TTI::MaskSource Mask,
std::optional<Instruction::CastOps> CastHint) const override;
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5164c6de2fc8d..af87e84f9c92d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5792,9 +5792,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
*PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
- if (TTI.getMaximumVFMultipleForMemoryOp(BestVF, BestUF) > 1)
- RUN_VPLAN_PASS(VPlanTransforms::widenMemoryAccessesToVFMultiple, BestVPlan,
- BestVF, BestUF, TTI);
+ if (TTI.hasMultipleVectorLoadStore(BestUF))
+ RUN_VPLAN_PASS(VPlanTransforms::widenMemoryAccessesByUF, BestVPlan, BestVF,
+ BestUF, TTI);
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index dcf15a8c3102d..dd1462074899f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1317,16 +1317,15 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// WideActiveLaneMask is used for control flow and is unrolled by widening,
// with one extract vector created per unroll part.
WideActiveLaneMask,
- // Signature: (VFMultiple, Address, Alignment) -> Wide Vector
- // Loads a single wide vector of `VFMultiple * VF` elements. VFMultiple must
- // divide UF. After unrolling, each section of VF elements in the wide
- // vector corresponds to an unroll part.
- VFMultipleLoad,
- // Signature: (VFMultiple, Address, Alignment, Vectors...)
- // Concatenates VFMultiple vector operands into a single wide vector of
- // `VFMultiple * VF` elements and stores it. After unrolling, each vector
- // operand corresponds to an unroll part.
- VFMultipleStore,
+ // Signature: Vectors... -> WideVector
+ // Concatenates all vector operands to a single wide vector.
+ ConcatVectors,
+ // Signature: (Multiplier, Address, Align) -> Vector
+ // Loads a single wide vector of `Multiplier * VF` elements.
+ WideVectorLoad,
+ // Signature: (Multiplier, Address, Alignment, Vector)
+ // Stores a single wide vector of `Multiplier * VF` elements.
+ WideVectorStore,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
ExplicitVectorLength,
@@ -1531,7 +1530,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
case VPInstruction::BranchOnCond:
case VPInstruction::BranchOnTwoConds:
case VPInstruction::BranchOnCount:
- case VPInstruction::VFMultipleStore:
+ case VPInstruction::WideVectorStore:
return false;
default:
return true;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 947f1a20e648f..0b8e66966ad90 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -60,7 +60,7 @@ bool VPRecipeBase::mayWriteToMemory() const {
auto *VPI = cast<VPInstruction>(this);
// Loads read from memory but don't write to memory.
if (VPI->getOpcode() == Instruction::Load ||
- VPI->getOpcode() == VPInstruction::VFMultipleLoad)
+ VPI->getOpcode() == VPInstruction::WideVectorLoad)
return false;
return VPI->opcodeMayReadOrWriteFromMemory();
}
@@ -122,7 +122,7 @@ bool VPRecipeBase::mayReadFromMemory() const {
case VPInstructionSC: {
auto *VPI = cast<VPInstruction>(this);
// Stores write to memory but don't read from memory.
- if (VPI->getOpcode() == VPInstruction::VFMultipleStore)
+ if (VPI->getOpcode() == VPInstruction::WideVectorStore)
return false;
return VPI->opcodeMayReadOrWriteFromMemory();
}
@@ -497,7 +497,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
for (unsigned Idx = 1; Idx != Operands.size(); ++Idx)
AssertOperandType(Idx, Op0Ty);
return Type::getVoidTy(Ctx);
- case VPInstruction::VFMultipleStore:
+ case VPInstruction::WideVectorStore:
case Instruction::Store:
return Type::getVoidTy(Ctx);
case Instruction::ICmp:
@@ -573,7 +573,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
}
case VPInstruction::ExtractVectorForPart:
return Op0Ty;
- case VPInstruction::VFMultipleLoad:
+ case VPInstruction::WideVectorLoad:
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::NumActiveLanes:
@@ -595,7 +595,8 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
Instruction::isBinaryOp(Opcode) ||
is_contained({VPInstruction::FirstOrderRecurrenceSplice,
VPInstruction::BuildVector,
- VPInstruction::BuildStructVector},
+ VPInstruction::BuildStructVector,
+ VPInstruction::ConcatVectors},
Opcode);
if (AllOperandsSameType)
for (unsigned Idx = 1; Idx != Operands.size(); ++Idx)
@@ -682,8 +683,10 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::Select:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
- case VPInstruction::VFMultipleLoad:
+ case VPInstruction::WideVectorLoad:
return 3;
+ case VPInstruction::WideVectorStore:
+ return 4;
case Instruction::Call:
return getCalledFnOperandIndex(operands()) + 1;
case Instruction::GetElementPtr:
@@ -702,7 +705,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLane:
case VPInstruction::ExtractLastActive:
- case VPInstruction::VFMultipleStore:
+ case VPInstruction::ConcatVectors:
// Cannot determine the number of operands from the opcode.
return -1u;
}
@@ -963,6 +966,15 @@ Value *VPInstruction::generate(VPTransformState &State,
Builder.getInt64(Idx));
return Res;
}
+ case VPInstruction::ConcatVectors: {
+ Type *ScalarTy = getScalarType();
+ auto *WideTy = VectorType::get(ScalarTy, State.VF * getNumOperands());
+ Value *Res = PoisonValue::get(WideTy);
+ for (const auto &[Idx, Op] : enumerate(operands()))
+ Res = Builder.CreateInsertVector(WideTy, Res, State.get(Op),
+ Idx * State.VF.getKnownMinValue());
+ return Res;
+ }
case VPInstruction::ReductionStartVector: {
if (State.VF.isScalar())
return State.get(getOperand(0), true);
@@ -1192,26 +1204,21 @@ Value *VPInstruction::generate(VPTransformState &State,
vputils::getIntrinsicID(this), Args,
/*FMFSource=*/nullptr, getName());
}
- case VPInstruction::VFMultipleLoad: {
- unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
- auto *WideDataTy = VectorType::get(getScalarType(), State.VF * VFMultiple);
+ case VPInstruction::WideVectorLoad: {
+ unsigned Multiplier = cast<VPConstantInt>(getOperand(0))->getZExtValue();
+ auto *WideDataTy = VectorType::get(getScalarType(), State.VF * Multiplier);
Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
Align Alignment = Align(cast<VPConstantInt>(getOperand(2))->getZExtValue());
- return Builder.CreateAlignedLoad(WideDataTy, Addr, Alignment,
- "vf.multiple.load");
- }
- case VPInstruction::VFMultipleStore: {
- unsigned VFMultiple = cast<VPConstantInt>(getOperand(0))->getZExtValue();
- Type *ScalarStoreTy = getOperand(3)->getScalarType();
- auto *WideDataTy = VectorType::get(ScalarStoreTy, State.VF * VFMultiple);
-
- Value *WideData = PoisonValue::get(WideDataTy);
- for (unsigned I = 0; I < VFMultiple; ++I) {
- Value *Part = State.get(getOperand(I + 3));
- WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part,
- I * State.VF.getKnownMinValue());
- }
+ return Builder.CreateAlignedLoad(WideDataTy, Addr, Alignment);
+ }
+ case VPInstruction::WideVectorStore: {
+ unsigned Multiplier = cast<VPConstantInt>(getOperand(0))->getZExtValue();
+ auto *WideDataTy =
+ VectorType::get(getOperand(3)->getScalarType(), State.VF * Multiplier);
+ Value *WideData = State.get(getOperand(3));
+ assert(WideData->getType() == WideDataTy &&
+ "stored value does not match wide vector type");
Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
Align Alignment = Align(cast<VPConstantInt>(getOperand(2))->getZExtValue());
@@ -1680,6 +1687,7 @@ void VPInstruction::addOperand(VPValue *Op) {
case VPInstruction::ComputeReductionResult:
case VPInstruction::BuildVector:
case VPInstruction::BuildStructVector:
+ case VPInstruction::ConcatVectors:
assert(Ty == getOperand(0)->getScalarType() &&
"appended operand must match operand 0's scalar type");
break;
@@ -1700,10 +1708,6 @@ void VPInstruction::addOperand(VPValue *Op) {
"matching operand 1's type and i1, respectively");
break;
}
- case VPInstruction::VFMultipleStore:
- assert(Ty == getOperand(3)->getScalarType() &&
- "appended operand must match operand 3's scalar type");
- break;
default:
llvm_unreachable("opcode does not support growing the operand list "
"outside of construction");
@@ -1761,6 +1765,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::Broadcast:
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
+ case VPInstruction::ConcatVectors:
case VPInstruction::CanonicalIVIncrementForPart:
case VPInstruction::ComputeReductionResult:
case VPInstruction::ExtractLane:
@@ -1838,7 +1843,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::Intrinsic:
case VPInstruction::ReductionStartVector:
case VPInstruction::ResumeForEpilogue:
- case VPInstruction::VFMultipleLoad:
+ case VPInstruction::WideVectorLoad:
return true;
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
@@ -1851,7 +1856,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::WidePtrAdd:
// WidePtrAdd supports scalar and vector base addresses.
return false;
- case VPInstruction::VFMultipleStore:
+ case VPInstruction::WideVectorStore:
return Op == getOperand(0) || Op == getOperand(1) || Op == getOperand(2);
case VPInstruction::ExitingIVValue:
case VPInstruction::ExtractLane:
@@ -1906,11 +1911,14 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::WideActiveLaneMask:
O << "wide active lane mask";
break;
- case VPInstruction::VFMultipleLoad:
- O << "vf-multiple load";
+ case VPInstruction::WideVectorLoad:
+ O << "wide vector load";
+ break;
+ case VPInstruction::WideVectorStore:
+ O << "wide vector store";
break;
- case VPInstruction::VFMultipleStore:
- O << "vf-multiple store";
+ case VPInstruction::ConcatVectors:
+ O << "concat-vectors";
break;
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ffd41692dd1b6..4174afe673ab2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4196,19 +4196,17 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
}
}
-void VPlanTransforms::widenMemoryAccessesToVFMultiple(
- VPlan &Plan, ElementCount VF, unsigned UF, const TargetTransformInfo &TTI) {
+void VPlanTransforms::widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
+ unsigned UF,
+ const TargetTransformInfo &TTI) {
assert(UF > 1 && "Expected plan to have an UF > 1");
-
- Type *I64Ty = IntegerType::getInt64Ty(Plan.getContext());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
VPValue *StoredValue = nullptr;
auto m_ContiguousVecPtr = m_VecPtr(m_VPValue(), m_One());
- if ((!match(&R, m_WidenLoad(m_ContiguousVecPtr)) &&
- !match(&R,
- m_WidenStore(m_ContiguousVecPtr, m_VPValue(StoredValue)))))
+ if (!match(&R, m_WidenLoad(m_ContiguousVecPtr)) &&
+ !match(&R, m_WidenStore(m_ContiguousVecPtr, m_VPValue(StoredValue))))
continue;
auto *MemOp = cast<VPWidenMemoryRecipe>(&R);
@@ -4220,41 +4218,40 @@ void VPlanTransforms::widenMemoryAccessesToVFMultiple(
Type *AccessType = StoredValue ? StoredValue->getScalarType()
: R.getVPSingleValue()->getScalarType();
- unsigned Opcode = isa<VPWidenLoadRecipe>(MemOp->getAsRecipe())
- ? Instruction::Load
- : Instruction::Store;
-
+ unsigned IsStore = isa<VPWidenStoreRecipe>(MemOp->getAsRecipe());
std::optional<Instruction::CastOps> CastHint;
- VPUser *MaybeCast = Opcode == Instruction::Store
- ? StoredValue->getDefiningRecipe()
- : R.getVPSingleValue()->getSingleUser();
+ VPUser *MaybeCast = IsStore ? StoredValue->getDefiningRecipe()
+ : R.getVPSingleValue()->getSingleUser();
if (auto *Cast = dyn_cast_if_present<VPWidenCastRecipe>(MaybeCast))
CastHint = Cast->getOpcode();
- unsigned VFMultiple = TTI.getPreferredVFMultipleForMemoryOp(
- Opcode, AccessType, VF, UF, /*IsMasked=*/false, CastHint);
- assert((VFMultiple != 0 && UF % VFMultiple == 0) &&
- "VFMultiple must divide UF");
-
- if (VFMultiple == 1)
+ VectorType *VectorAccessType = VectorType::get(AccessType, VF);
+ if (!TTI.hasMultipleVectorLoadStore(
+ /*NumVectors=*/UF, VectorAccessType, IsStore,
+ TargetTransformInfo::MaskSource::MS_None, CastHint))
continue;
+ DebugLoc DL = R.getDebugLoc();
VPValue *Ptr = MemOp->getAddr();
- VPValue *VFMultipleVPV = Plan.getConstantInt(I64Ty, VFMultiple);
- VPValue *Align = Plan.getConstantInt(I64Ty, MemOp->getAlign().value());
+ VPValue *Align = Plan.getConstantInt(64, MemOp->getAlign().value());
+ VPValue *Multiplier = Plan.getConstantInt(64, 1);
VPBuilder Builder(VPBB, R.getIterator());
- if (Opcode == Instruction::Load) {
+ if (IsStore) {
+ VPValue *WideStoredValue = Builder.createNaryOp(
+ VPInstruction::ConcatVectors, {StoredValue}, DL);
+ Builder.createNaryOp(VPInstruction::WideVectorStore,
+ {Multiplier, Ptr, Align, WideStoredValue}, nullptr,
+ {}, *MemOp, R.getDebugLoc());
+ } else {
VPValue *OldLoad = R.getVPSingleValue();
VPValue *Load = Builder.createNaryOp(
- VPInstruction::VFMultipleLoad, {VFMultipleVPV, Ptr, Align}, nullptr,
+ VPInstruction::WideVectorLoad, {Multiplier, Ptr, Align}, nullptr,
{}, *MemOp, R.getDebugLoc(), "", OldLoad->getScalarType());
- OldLoad->replaceAllUsesWith(Load);
- } else {
- assert(Opcode == Instruction::Store);
- Builder.createNaryOp(VPInstruction::VFMultipleStore,
- {VFMultipleVPV, Ptr, Align, StoredValue}, nullptr,
- {}, *MemOp, R.getDebugLoc());
+ VPValue *Extract =
+ Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
+ {Load, Plan.getConstantInt(64, 0)}, DL);
+ OldLoad->replaceAllUsesWith(Extract);
}
R.eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index d4f246b13e2a2..64f66c1b351ae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -484,9 +484,8 @@ struct VPlanTransforms {
/// Widens memory operations by a factor of UF based on a target hook.
/// This allows targets to use wider memory operations when profitable.
- static void widenMemoryAccessesToVFMultiple(VPlan &Plan, ElementCount VF,
- unsigned UF,
- const TargetTransformInfo &TTI);
+ static void widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF, unsigned UF,
+ const TargetTransformInfo &TTI);
// Materialize vector trip counts for constants early if it can simply be
// computed as (Original TC / VF * UF) * VF * UF.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index d442d407938bd..23a34b1fd831c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -74,9 +74,6 @@ class UnrollState {
return Plan.getConstantInt(CanIVIntTy, Part);
}
- /// Unroll a VFMultipleLoad or VFMultipleStore VPInstruction.
- void unrollMemOpWithVFMultiple(VPInstruction *VPI);
-
public:
UnrollState(VPlan &Plan, unsigned UF) : Plan(Plan), UF(UF) {}
@@ -293,76 +290,12 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
}
}
-void UnrollState::unrollMemOpWithVFMultiple(VPInstruction *VPI) {
- assert((VPI->getOpcode() == VPInstruction::VFMultipleLoad ||
- VPI->getOpcode() == VPInstruction::VFMultipleStore) &&
- "expected a vf-multiple load/store");
-
- unsigned VFMultiple = cast<VPConstantInt>(VPI->getOperand(0))->getZExtValue();
- assert(VFMultiple > 1 && UF % VFMultiple == 0 &&
- "expected VFMultiple to divide UF");
-
- SmallVector<VPInstruction *, 4> Groups(UF / VFMultiple, nullptr);
- Groups[0] = VPI;
-
- // A memory op with a VFMultiple is widened to VF * VFMultiple elements, so
- // after unrolling by UF we materialize UF / VFMultiple such ops, each
- // covering VFMultiple unroll parts.
- VPBuilder Builder = VPBuilder::getToInsertAfter(VPI);
- for (unsigned Group = 1; Group < Groups.size(); ++Group) {
- auto *Copy = Builder.insert(VPI->clone());
- remapOperands(Copy, Group * VFMultiple);
- Groups[Group] = Copy;
- }
-
- if (VPI->getOpcode() == VPInstruction::VFMultipleStore) {
- VPValue *StoredValue = VPI->getOperand(3);
- for (unsigned Group = 0; Group < Groups.size(); ++Group) {
- VPInstruction *Store = Groups[Group];
- // Add the value to store for each unroll part in this group.
- for (unsigned Part = 0; Part < VFMultiple; ++Part) {
- VPValue *UnrollPart =
- getValueForPart(StoredValue, Group * VFMultiple + Part);
- if (Part == 0)
- Store->setOperand(3, UnrollPart);
- else
- Store->addOperand(UnrollPart);
- }
- }
- return;
- }
-
- // We need to extract each unroll part as a subvector.
- auto *ExtractPart0 = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
- {Groups[0], getConstantInt(0)});
- // First VPI with an extract of the first unroll part (ExtractPart0).
- VPI->replaceUsesWithIf(
- ExtractPart0, [&](VPUser &U, unsigned) { return &U != ExtractPart0; });
-
- // Create extracts for the remaining unroll parts and remap later uses of
- // ExtractPart0 to the correct unrolled part.
- for (unsigned Part = 1; Part != UF; ++Part) {
- VPInstruction *Group = Groups[Part / VFMultiple];
- unsigned IndexInGroup = Part % VFMultiple;
- auto *Extract = Builder.createNaryOp(
- VPInstruction::ExtractVectorForPart,
- {Group->getVPSingleValue(), getConstantInt(IndexInGroup)});
- addRecipeForPart(ExtractPart0, Extract, Part);
- }
-}
-
/// Handle non-header-phi recipes.
void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
if (match(&R, m_CombineOr(m_BranchOnCond(), m_BranchOnCount())))
return;
if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
- if (VPI->getOpcode() == VPInstruction::VFMultipleLoad ||
- VPI->getOpcode() == VPInstruction::VFMultipleStore) {
- unrollMemOpWithVFMultiple(VPI);
- return;
- }
-
if (vputils::onlyFirstPartUsed(VPI)) {
addUniformForAllParts(VPI);
return;
@@ -496,6 +429,8 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
// value.
VPValue *Op1;
if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) ||
+ match(&R,
+ m_VPInstruction<VPInstruction::ConcatVectors>(m_VPValue(Op1))) ||
match(&R, m_FirstActiveLane(m_VPValue(Op1))) ||
match(&R, m_LastActiveLane(m_VPValue(Op1))) ||
match(&R, m_ComputeReductionResult(m_VPValue(Op1)))) {
@@ -553,6 +488,13 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
+ if (match(&R,
+ m_CombineOr(m_VPInstruction<VPInstruction::WideVectorLoad>(),
+ m_VPInstruction<VPInstruction::WideVectorStore>()))) {
+ cast<VPInstruction>(&R)->setOperand(0, Plan.getConstantInt(64, UF));
+ continue;
+ }
+
auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);
if (SingleDef && vputils::isUniformAcrossVFsAndUFs(SingleDef)) {
addUniformForAllParts(SingleDef);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
index 529909d353e17..4d2ecb32f9326 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops-non-power-of-two-unroll.ll
@@ -3,7 +3,7 @@
target triple = "aarch64-unknown-linux-gnu"
-; Tests `widenMemoryAccessesToVFMultiple` with a non-power-of-two unroll factor.
+; Tests `widenMemoryAccessesByUF` with a non-power-of-two unroll factor.
; On AArch64, this should be rejected (which results in `vscale x 4` loads/stores).
define void @mixed_i64_i32_accesses(ptr noalias %x, ptr noalias %y, i64 %n) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index 3b41923deada9..f7c2390e258a9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -16,46 +16,41 @@ define void @mixed_i64_i32_accesses(ptr noalias %x, ptr noalias %y, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP11]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP15]], ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP13]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP17]], ptr [[TMP5]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8
; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP24]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP25]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP30]], ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP32]], <vscale x 4 x i32> [[TMP24]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -95,52 +90,47 @@ define void @mixed_i32_more_frequent_than_i64(ptr noalias %x, ptr noalias %y, pt
; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD2]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP11]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP15]], ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP13]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP17]], ptr [[TMP5]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8
; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP24]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP25]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP29]], <vscale x 4 x i32> [[TMP26]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP30]], ptr [[TMP18]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP45:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP45]], <vscale x 4 x i32> [[TMP24]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4
; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i32>, ptr [[TMP31]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD4]], i64 12)
+; UNMASKED-SVE2P1-NEXT: [[TMP46:%.*]] = load <vscale x 16 x i32>, ptr [[TMP31]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 12)
; UNMASKED-SVE2P1-NEXT: [[TMP36:%.*]] = add <vscale x 4 x i32> [[TMP32]], splat (i32 2)
; UNMASKED-SVE2P1-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[TMP33]], splat (i32 2)
; UNMASKED-SVE2P1-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], splat (i32 2)
@@ -262,11 +252,11 @@ define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
+; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 6)
; UNMASKED-SVE2P1-NEXT: [[TMP8]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP4]]
; UNMASKED-SVE2P1-NEXT: [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP5]]
; UNMASKED-SVE2P1-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP6]]
@@ -318,11 +308,11 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP6]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]], i64 6)
+; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 8 x i64>, ptr [[TMP6]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 2)
+; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 6)
; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP7]]
; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP8]]
; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
@@ -405,13 +395,11 @@ define void @extending_load(ptr noalias %dst, ptr %src, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = mul nsw <vscale x 4 x i64> [[TMP11]], splat (i64 42)
; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = mul nsw <vscale x 4 x i64> [[TMP12]], splat (i64 42)
; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[DST]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP17]], i64 [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP13]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP19]], <vscale x 4 x i64> [[TMP14]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP20]], ptr [[TMP17]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP15]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP21]], <vscale x 4 x i64> [[TMP16]], i64 4)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x i64> [[TMP22]], ptr [[TMP18]], align 8
+; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP13]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP14]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP19]], <vscale x 4 x i64> [[TMP15]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP20]], <vscale x 4 x i64> [[TMP16]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i64> [[TMP21]], ptr [[TMP17]], align 8
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
@@ -573,11 +561,11 @@ define void @gather_nxv4i32_stride2(ptr noalias %a, ptr noalias %b, i64 %n) {
; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC6]])
; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC7]], 0
; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP24]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP29]], <vscale x 4 x float> [[TMP25]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP30]], <vscale x 4 x float> [[TMP26]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP31]], <vscale x 4 x float> [[TMP27]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x float> [[TMP32]], ptr [[TMP28]], align 4
+; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP24]], i64 0)
+; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP30]], <vscale x 4 x float> [[TMP25]], i64 4)
+; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP31]], <vscale x 4 x float> [[TMP26]], i64 8)
+; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP32]], <vscale x 4 x float> [[TMP27]], i64 12)
+; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x float> [[TMP29]], ptr [[TMP28]], align 4
; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
index 82b1201c22629..05464ccc940a4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
-; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-before="widenMemoryAccessesToVFMultiple$" 2>&1 \
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-before="widenMemoryAccessesByUF$" 2>&1 \
; RUN: | FileCheck --check-prefix=BEFORE-SCALE %s
-; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="widenMemoryAccessesToVFMultiple$" 2>&1 \
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="widenMemoryAccessesByUF$" 2>&1 \
; RUN: | FileCheck --check-prefix=AFTER-SCALE %s
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-width="vscale x 4" -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -disable-output -S %s -vplan-print-after="unrollByUF$" 2>&1 \
; RUN: | FileCheck --check-prefix=AFTER-UNROLL %s
@@ -69,10 +69,12 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-SCALE-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; AFTER-SCALE-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
; AFTER-SCALE-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-SCALE-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP9]]>, ir<8>
-; AFTER-SCALE-NEXT: WIDEN ir<%add> = add vp<[[VP10]]>, ir<1>
-; AFTER-SCALE-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-SCALE-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP11]]>, ir<8>, ir<%add>
+; AFTER-SCALE-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = wide vector load ir<1>, vp<[[VP9]]>, ir<8>
+; AFTER-SCALE-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-vector-for-part vp<[[VP10]]>, ir<0>
+; AFTER-SCALE-NEXT: WIDEN ir<%add> = add vp<[[VP11]]>, ir<1>
+; AFTER-SCALE-NEXT: vp<[[VP12:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-SCALE-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = concat-vectors ir<%add>
+; AFTER-SCALE-NEXT: EMIT wide vector store ir<1>, vp<[[VP12]]>, ir<8>, vp<[[VP13]]>
; AFTER-SCALE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
; AFTER-SCALE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; AFTER-SCALE-NEXT: No successors
@@ -105,24 +107,19 @@ define void @i64_load_store(ptr noalias %x, i64 %n) {
; AFTER-UNROLL-NEXT: vector.body:
; AFTER-UNROLL-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; AFTER-UNROLL-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%x>, vp<[[VP8]]>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
-; AFTER-UNROLL-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-UNROLL-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP9]]>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP10]]>, ir<8>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = vf-multiple load ir<2>, vp<[[VP11]]>, ir<8>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<0>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-vector-for-part vp<[[VP12]]>, ir<1>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-vector-for-part vp<[[VP13]]>, ir<0>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-vector-for-part vp<[[VP13]]>, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add> = add vp<[[VP14]]>, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add>.1 = add vp<[[VP15]]>, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add>.2 = add vp<[[VP16]]>, ir<1>
-; AFTER-UNROLL-NEXT: WIDEN ir<%add>.3 = add vp<[[VP17]]>, ir<1>
-; AFTER-UNROLL-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<2>
-; AFTER-UNROLL-NEXT: vp<[[VP19:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
-; AFTER-UNROLL-NEXT: vp<[[VP20:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>, vp<[[VP18]]>
-; AFTER-UNROLL-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP19]]>, ir<8>, ir<%add>, ir<%add>.1
-; AFTER-UNROLL-NEXT: EMIT vf-multiple store ir<2>, vp<[[VP20]]>, ir<8>, ir<%add>.2, ir<%add>.3
+; AFTER-UNROLL-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = wide vector load ir<4>, vp<[[VP9]]>, ir<8>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-vector-for-part vp<[[VP10]]>, ir<0>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-vector-for-part vp<[[VP10]]>, ir<1>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-vector-for-part vp<[[VP10]]>, ir<2>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-vector-for-part vp<[[VP10]]>, ir<3>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add> = add vp<[[VP11]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.1 = add vp<[[VP12]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.2 = add vp<[[VP13]]>, ir<1>
+; AFTER-UNROLL-NEXT: WIDEN ir<%add>.3 = add vp<[[VP14]]>, ir<1>
+; AFTER-UNROLL-NEXT: vp<[[VP15:%[0-9]+]]> = vector-pointer inbounds i64, ir<%ptr>, ir<1>
+; AFTER-UNROLL-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = concat-vectors ir<%add>, ir<%add>.1, ir<%add>.2, ir<%add>.3
+; AFTER-UNROLL-NEXT: EMIT wide vector store ir<4>, vp<[[VP15]]>, ir<8>, vp<[[VP16]]>
; AFTER-UNROLL-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
; AFTER-UNROLL-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; AFTER-UNROLL-NEXT: No successors
>From 3fa986ae39abcea4d12b87d16d3103132d437a33 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 1 Oct 2026 12:17:35 +0000
Subject: [PATCH 18/21] Rebase fixups
---
.../AArch64/multi-vector-mem-ops.ll | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index f7c2390e258a9..e0b7897250634 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -303,10 +303,10 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 8 x i64>, ptr [[TMP6]], align 8
; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 0)
@@ -325,10 +325,10 @@ define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold)
; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = or <vscale x 2 x i1> [[TMP32]], [[TMP18]]
; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = or <vscale x 2 x i1> [[TMP19]], [[TMP20]]
; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP21]])
-; UNMASKED-SVE2P1-NEXT: [[TMP23]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP24]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> [[TMP3]]
-; UNMASKED-SVE2P1-NEXT: [[TMP25]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP13]], <vscale x 2 x i1> [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP26]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> [[TMP5]]
+; UNMASKED-SVE2P1-NEXT: [[TMP24]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP2]]
+; UNMASKED-SVE2P1-NEXT: [[TMP25]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i1> [[TMP3]]
+; UNMASKED-SVE2P1-NEXT: [[TMP26]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP18]], <vscale x 2 x i1> [[TMP4]]
+; UNMASKED-SVE2P1-NEXT: [[TMP33]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP20]], <vscale x 2 x i1> [[TMP5]]
; UNMASKED-SVE2P1-NEXT: [[TMP27]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[VEC_PHI]]
; UNMASKED-SVE2P1-NEXT: [[TMP28]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[VEC_PHI1]]
; UNMASKED-SVE2P1-NEXT: [[TMP29]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI2]]
>From 4a7ef8a27ce94f548dd6a0bd7ab728a581e490e6 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 2 Oct 2026 08:55:56 +0000
Subject: [PATCH 19/21] Fixups
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 14 +++++++-------
.../llvm/Analysis/TargetTransformInfoImpl.h | 8 ++++----
llvm/lib/Analysis/TargetTransformInfo.cpp | 10 +++++-----
.../Target/AArch64/AArch64TargetTransformInfo.cpp | 12 ++++++------
.../Target/AArch64/AArch64TargetTransformInfo.h | 6 +++---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 +++++---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 +++---
7 files changed, 33 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 91750f5ec5652..7fc4f730275e3 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1001,13 +1001,13 @@ class TargetTransformInfo {
unsigned Opcode1,
const SmallBitVector &OpcodeMask) const;
- enum MaskSource {
+ enum class MaskSource {
/// The operation is unmasked.
- MS_None,
+ None,
/// The operation is masked with an arbitrary predicate.
- MS_Masked,
+ ArbitraryPredicate,
/// The operation is masked with a contiguous active lane mask.
- MS_ActiveLaneMask,
+ ActiveLaneMask,
};
/// Return true if the target supports loading or storing \p NumVectors
@@ -1017,9 +1017,9 @@ class TargetTransformInfo {
///
/// \p CastHint is non-null if a stored value is produced by a cast or a
/// loaded value is consumed by one.
- LLVM_ABI bool hasMultipleVectorLoadStore(
- unsigned NumVectors, VectorType *VectorTy = nullptr, bool IsStore = false,
- MaskSource Mask = MS_None,
+ LLVM_ABI bool hasMultiVectorLoadStore(
+ unsigned NumVectors, MaskSource Mask, VectorType *VectorTy = nullptr,
+ bool IsStore = false,
std::optional<Instruction::CastOps> CastHint = std::nullopt) const;
/// Return true if we should be enabling ordered reductions for the target.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index cd4acc0731c7d..004602490fbbd 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -429,10 +429,10 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
- virtual bool hasMultipleVectorLoadStore(
- unsigned NumVectors, VectorType *VectorTy, bool IsStore,
- TTI::MaskSource Mask,
- std::optional<Instruction::CastOps> CastHint) const {
+ virtual bool
+ hasMultiVectorLoadStore(unsigned NumVectors, TTI::MaskSource Mask,
+ VectorType *VectorTy, bool IsStore,
+ std::optional<Instruction::CastOps> CastHint) const {
return false;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 8e3bb78b71218..957384b83233b 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -554,11 +554,11 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
-bool TargetTransformInfo::hasMultipleVectorLoadStore(
- unsigned NumVectors, VectorType *VectorTy, bool IsStore,
- TTI::MaskSource Mask, std::optional<Instruction::CastOps> CastHint) const {
- return TTIImpl->hasMultipleVectorLoadStore(NumVectors, VectorTy, IsStore,
- Mask, CastHint);
+bool TargetTransformInfo::hasMultiVectorLoadStore(
+ unsigned NumVectors, TTI::MaskSource Mask, VectorType *VectorTy,
+ bool IsStore, std::optional<Instruction::CastOps> CastHint) const {
+ return TTIImpl->hasMultiVectorLoadStore(NumVectors, Mask, VectorTy, IsStore,
+ CastHint);
}
bool TargetTransformInfo::isLegalInterleavedAccessType(
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8954aab4cb781..3435a90714c0c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5974,14 +5974,14 @@ bool AArch64TTIImpl::isLegalSpeculativeLoad(Type *DataType,
Size.getFixedValue() <= 16;
}
-bool AArch64TTIImpl::hasMultipleVectorLoadStore(
- unsigned NumVectors, VectorType *VectorTy, bool IsStore,
- TTI::MaskSource Mask, std::optional<Instruction::CastOps> CastHint) const {
+bool AArch64TTIImpl::hasMultiVectorLoadStore(
+ unsigned NumVectors, TTI::MaskSource Mask, VectorType *VectorTy,
+ bool IsStore, std::optional<Instruction::CastOps> CastHint) const {
if (NumVectors <= 1 || !ST->enableSubRegLiveness() || !ST->hasSVE2p1())
return false;
// TODO: Support masked multi-vector loads/stores.
- if (Mask != TTI::MS_None)
+ if (Mask != TTI::MaskSource::None)
return false;
// A null vector type queries whether the target supports multi-vector memory
@@ -6002,8 +6002,8 @@ bool AArch64TTIImpl::hasMultipleVectorLoadStore(
// For unpredicated loads/stores allow any pow-of-two multiple of a vector >=
// to a single z-register. We can split operations wider than a single
// multi-vector load/store during ISEL.
- unsigned VectorWidth = DL.getTypeSizeInBits(VectorTy).getKnownMinValue();
- return VectorWidth % 128 == 0 && isPowerOf2_32(NumVectors);
+ return DL.getTypeSizeInBits(VectorTy).isKnownMultipleOf(128) &&
+ isPowerOf2_32(NumVectors);
}
unsigned
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 010b1f9142edd..b1aec1c391eff 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -282,9 +282,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isLegalSpeculativeLoad(Type *DataType,
unsigned AddressSpace) const override;
- bool hasMultipleVectorLoadStore(
- unsigned NumVectors, VectorType *VectorTy, bool IsStore,
- TTI::MaskSource Mask,
+ bool hasMultiVectorLoadStore(
+ unsigned NumVectors, TTI::MaskSource Mask, VectorType *VectorTy,
+ bool IsStore,
std::optional<Instruction::CastOps> CastHint) const override;
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index af87e84f9c92d..79098307e5f57 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5790,11 +5790,13 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::replaceWideCanonicalIVWithWideIV, BestVPlan,
*PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
- // TODO: Move to VPlan transform stage once the transition to the VPlan-based
- // cost model is complete for better cost estimates.
- if (TTI.hasMultipleVectorLoadStore(BestUF))
+ if (!BestVPlan.hasTailFolded() &&
+ TTI.hasMultiVectorLoadStore(BestUF,
+ TargetTransformInfo::MaskSource::None))
RUN_VPLAN_PASS(VPlanTransforms::widenMemoryAccessesByUF, BestVPlan, BestVF,
BestUF, TTI);
+ // TODO: Move to VPlan transform stage once the transition to the VPlan-based
+ // cost model is complete for better cost estimates.
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4174afe673ab2..00bc58d2057f6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4226,9 +4226,9 @@ void VPlanTransforms::widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
CastHint = Cast->getOpcode();
VectorType *VectorAccessType = VectorType::get(AccessType, VF);
- if (!TTI.hasMultipleVectorLoadStore(
- /*NumVectors=*/UF, VectorAccessType, IsStore,
- TargetTransformInfo::MaskSource::MS_None, CastHint))
+ if (!TTI.hasMultiVectorLoadStore(
+ /*NumVectors=*/UF, TargetTransformInfo::MaskSource::None,
+ VectorAccessType, IsStore, CastHint))
continue;
DebugLoc DL = R.getDebugLoc();
>From 22c7c8cd5fbefa0454b89cc5504bbfedd5b9ad97 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 2 Oct 2026 08:59:03 +0000
Subject: [PATCH 20/21] Add comment
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 7fc4f730275e3..f47cc3888dd8a 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1001,6 +1001,7 @@ class TargetTransformInfo {
unsigned Opcode1,
const SmallBitVector &OpcodeMask) const;
+ /// Enum describing the source/producer of a mask.
enum class MaskSource {
/// The operation is unmasked.
None,
>From d44b99f6dc96f6c2a0828d4e58738537c6d65b41 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 6 Oct 2026 09:51:47 +0000
Subject: [PATCH 21/21] Fixups
---
.../AArch64/AArch64TargetTransformInfo.cpp | 5 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 18 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 5 +-
.../AArch64/multi-vector-mem-ops.ll | 905 +++++++++---------
4 files changed, 474 insertions(+), 459 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 3435a90714c0c..072e54e9752ee 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6002,8 +6002,9 @@ bool AArch64TTIImpl::hasMultiVectorLoadStore(
// For unpredicated loads/stores allow any pow-of-two multiple of a vector >=
// to a single z-register. We can split operations wider than a single
// multi-vector load/store during ISEL.
- return DL.getTypeSizeInBits(VectorTy).isKnownMultipleOf(128) &&
- isPowerOf2_32(NumVectors);
+ return isPowerOf2_32(NumVectors) &&
+ DL.getTypeSizeInBits(VectorTy).isKnownMultipleOf(
+ AArch64::SVEBitsPerBlock);
}
unsigned
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 0b8e66966ad90..b05726aa87786 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -571,8 +571,6 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
return StructTy->getTypeAtIndex(
cast<VPConstantInt>(Operands[1])->getZExtValue());
}
- case VPInstruction::ExtractVectorForPart:
- return Op0Ty;
case VPInstruction::WideVectorLoad:
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
@@ -1210,19 +1208,23 @@ Value *VPInstruction::generate(VPTransformState &State,
Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
Align Alignment = Align(cast<VPConstantInt>(getOperand(2))->getZExtValue());
- return Builder.CreateAlignedLoad(WideDataTy, Addr, Alignment);
+ LoadInst *WideLI = Builder.CreateAlignedLoad(WideDataTy, Addr, Alignment);
+ applyMetadata(*WideLI);
+ return WideLI;
}
case VPInstruction::WideVectorStore: {
unsigned Multiplier = cast<VPConstantInt>(getOperand(0))->getZExtValue();
- auto *WideDataTy =
- VectorType::get(getOperand(3)->getScalarType(), State.VF * Multiplier);
Value *WideData = State.get(getOperand(3));
- assert(WideData->getType() == WideDataTy &&
- "stored value does not match wide vector type");
+ assert(cast<VectorType>(WideData->getType())->getElementCount() ==
+ State.VF * Multiplier &&
+ "stored value does not match wide element count");
+ (void)Multiplier;
Value *Addr = State.get(getOperand(1), /*IsScalar=*/true);
Align Alignment = Align(cast<VPConstantInt>(getOperand(2))->getZExtValue());
- return Builder.CreateAlignedStore(WideData, Addr, Alignment);
+ StoreInst *WideSI = Builder.CreateAlignedStore(WideData, Addr, Alignment);
+ applyMetadata(*WideSI);
+ return WideSI;
}
default:
llvm_unreachable("Unsupported opcode for instruction");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 00bc58d2057f6..b4906d8ea879b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4200,11 +4200,12 @@ void VPlanTransforms::widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
unsigned UF,
const TargetTransformInfo &TTI) {
assert(UF > 1 && "Expected plan to have an UF > 1");
+
+ auto m_ContiguousVecPtr = m_VecPtr(m_VPValue(), m_One());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
VPValue *StoredValue = nullptr;
- auto m_ContiguousVecPtr = m_VecPtr(m_VPValue(), m_One());
if (!match(&R, m_WidenLoad(m_ContiguousVecPtr)) &&
!match(&R, m_WidenStore(m_ContiguousVecPtr, m_VPValue(StoredValue))))
continue;
@@ -4218,7 +4219,7 @@ void VPlanTransforms::widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
Type *AccessType = StoredValue ? StoredValue->getScalarType()
: R.getVPSingleValue()->getScalarType();
- unsigned IsStore = isa<VPWidenStoreRecipe>(MemOp->getAsRecipe());
+ bool IsStore = isa<VPWidenStoreRecipe>(MemOp->getAsRecipe());
std::optional<Instruction::CastOps> CastHint;
VPUser *MaybeCast = IsStore ? StoredValue->getDefiningRecipe()
: R.getVPSingleValue()->getSingleUser();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
index e0b7897250634..c34cf527d87a4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll
@@ -1,60 +1,71 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 6
-; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -sve-tail-folding=disabled %s | FileCheck %s --check-prefix=UNMASKED-SVE2P1
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -sve-tail-folding=disabled %s | FileCheck %s --check-prefix=CHECK
target triple = "aarch64-unknown-linux-gnu"
-define void @mixed_i64_i32_accesses(ptr noalias %x, ptr noalias %y, i64 %n) {
-; UNMASKED-SVE2P1-LABEL: define void @mixed_i64_i32_accesses(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP32]], <vscale x 4 x i32> [[TMP24]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; Note: `noalias` was not applied to the pointer arguments to test !noalias metadata on the loads/stores.
+define void @mixed_i64_i32_accesses(ptr %x, ptr %y, i64 %n) {
+; CHECK-LABEL: define void @mixed_i64_i32_accesses(
+; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ITER_CHECK:.*:]]
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP33:%.*]] = shl i64 [[TMP5]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[X]], i64 [[TMP33]]
+; CHECK-NEXT: [[TMP34:%.*]] = shl i64 [[TMP5]], 2
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[Y]], i64 [[TMP34]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[X]], [[TMP35]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Y]], [[TMP2]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], [[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
+; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
+; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
+; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; CHECK-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
+; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
+; CHECK-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8, !alias.scope [[META3]], !noalias [[META6]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4, !alias.scope [[META6]]
+; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
+; CHECK-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
+; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
+; CHECK-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
+; CHECK-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
+; CHECK-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
+; CHECK-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
+; CHECK-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
+; CHECK-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
+; CHECK-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP32]], <vscale x 4 x i32> [[TMP24]], i64 4)
+; CHECK-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
+; CHECK-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
+; CHECK-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4, !alias.scope [[META6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
@@ -78,72 +89,72 @@ exit:
}
define void @mixed_i32_more_frequent_than_i64(ptr noalias %x, ptr noalias %y, ptr noalias %z, i64 %n) {
-; UNMASKED-SVE2P1-LABEL: define void @mixed_i32_more_frequent_than_i64(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP45:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP45]], <vscale x 4 x i32> [[TMP24]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP46:%.*]] = load <vscale x 16 x i32>, ptr [[TMP31]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 12)
-; UNMASKED-SVE2P1-NEXT: [[TMP36:%.*]] = add <vscale x 4 x i32> [[TMP32]], splat (i32 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[TMP33]], splat (i32 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], splat (i32 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP39:%.*]] = add <vscale x 4 x i32> [[TMP35]], splat (i32 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP40:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP36]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP41:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP40]], <vscale x 4 x i32> [[TMP37]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP42:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP41]], <vscale x 4 x i32> [[TMP38]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP43:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP42]], <vscale x 4 x i32> [[TMP39]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i32> [[TMP43]], ptr [[TMP31]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define void @mixed_i32_more_frequent_than_i64(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*:]]
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
+; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
+; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
+; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
+; CHECK-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
+; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
+; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
+; CHECK-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
+; CHECK-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
+; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
+; CHECK-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
+; CHECK-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
+; CHECK-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
+; CHECK-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
+; CHECK-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
+; CHECK-NEXT: [[TMP45:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
+; CHECK-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP45]], <vscale x 4 x i32> [[TMP24]], i64 4)
+; CHECK-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
+; CHECK-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
+; CHECK-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP46:%.*]] = load <vscale x 16 x i32>, ptr [[TMP31]], align 4
+; CHECK-NEXT: [[TMP32:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 0)
+; CHECK-NEXT: [[TMP33:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 4)
+; CHECK-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 8)
+; CHECK-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 12)
+; CHECK-NEXT: [[TMP36:%.*]] = add <vscale x 4 x i32> [[TMP32]], splat (i32 2)
+; CHECK-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[TMP33]], splat (i32 2)
+; CHECK-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], splat (i32 2)
+; CHECK-NEXT: [[TMP39:%.*]] = add <vscale x 4 x i32> [[TMP35]], splat (i32 2)
+; CHECK-NEXT: [[TMP40:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP36]], i64 0)
+; CHECK-NEXT: [[TMP41:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP40]], <vscale x 4 x i32> [[TMP37]], i64 4)
+; CHECK-NEXT: [[TMP42:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP41]], <vscale x 4 x i32> [[TMP38]], i64 8)
+; CHECK-NEXT: [[TMP43:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP42]], <vscale x 4 x i32> [[TMP39]], i64 12)
+; CHECK-NEXT: store <vscale x 16 x i32> [[TMP43]], ptr [[TMP31]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
@@ -171,46 +182,46 @@ exit:
}
define void @first_order_recurrence_i64_scaled_load_and_store(ptr noalias %src, ptr noalias %dst) {
-; UNMASKED-SVE2P1-LABEL: define void @first_order_recurrence_i64_scaled_load_and_store(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[FIRST:%.*]] = load i64, ptr [[SRC]], align 8
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <2 x i64> poison, i64 [[FIRST]], i32 1
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD3:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = add nuw i64 1, [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 2
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 4
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 6
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3]] = load <2 x i64>, ptr [[TMP4]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[VECTOR_RECUR]], <2 x i64> [[WIDE_LOAD]], <2 x i32> <i32 1, i32 2>
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD]], <2 x i64> [[WIDE_LOAD1]], <2 x i32> <i32 1, i32 2>
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[WIDE_LOAD2]], <2 x i32> <i32 1, i32 2>
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD2]], <2 x i64> [[WIDE_LOAD3]], <2 x i32> <i32 1, i32 2>
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = add <2 x i64> [[TMP5]], [[WIDE_LOAD]]
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = add <2 x i64> [[TMP6]], [[WIDE_LOAD1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add <2 x i64> [[TMP7]], [[WIDE_LOAD2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = add <2 x i64> [[TMP8]], [[WIDE_LOAD3]]
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 2
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 4
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 6
-; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP13]], align 8
-; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP10]], ptr [[TMP14]], align 8
-; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP11]], ptr [[TMP15]], align 8
-; UNMASKED-SVE2P1-NEXT: store <2 x i64> [[TMP12]], ptr [[TMP16]], align 8
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define void @first_order_recurrence_i64_scaled_load_and_store(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[FIRST:%.*]] = load i64, ptr [[SRC]], align 8
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <2 x i64> poison, i64 [[FIRST]], i32 1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add nuw i64 1, [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 4
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 6
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[WIDE_LOAD3]] = load <2 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[VECTOR_RECUR]], <2 x i64> [[WIDE_LOAD]], <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD]], <2 x i64> [[WIDE_LOAD1]], <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[WIDE_LOAD2]], <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD2]], <2 x i64> [[WIDE_LOAD3]], <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i64> [[TMP5]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP10:%.*]] = add <2 x i64> [[TMP6]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP11:%.*]] = add <2 x i64> [[TMP7]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP12:%.*]] = add <2 x i64> [[TMP8]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 4
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 6
+; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP13]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP10]], ptr [[TMP14]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP11]], ptr [[TMP15]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP12]], ptr [[TMP16]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
%first = load i64, ptr %src, align 8
@@ -233,38 +244,38 @@ exit:
}
define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
-; UNMASKED-SVE2P1-LABEL: define i64 @i64_sum_reduction_scaled_partial_reduce(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 6)
-; UNMASKED-SVE2P1-NEXT: [[TMP8]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP5]]
-; UNMASKED-SVE2P1-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP6]]
-; UNMASKED-SVE2P1-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP7]]
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define i64 @i64_sum_reduction_scaled_partial_reduce(
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 0)
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 2)
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 6)
+; CHECK-NEXT: [[TMP8]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP4]]
+; CHECK-NEXT: [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP5]]
+; CHECK-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP6]]
+; CHECK-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
@@ -284,59 +295,59 @@ exit:
}
define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold) {
-; UNMASKED-SVE2P1-LABEL: define i64 @find_last_i64_scaled_load(
-; UNMASKED-SVE2P1-SAME: i64 [[N:%.*]], ptr noalias [[DATA:%.*]], i64 [[THRESHOLD:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[THRESHOLD]], i64 0
-; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 8 x i64>, ptr [[TMP6]], align 8
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 2)
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 6)
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP7]]
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP8]]
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = freeze <vscale x 2 x i1> [[TMP11]]
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = freeze <vscale x 2 x i1> [[TMP12]]
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = freeze <vscale x 2 x i1> [[TMP13]]
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = freeze <vscale x 2 x i1> [[TMP14]]
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = or <vscale x 2 x i1> [[TMP15]], [[TMP16]]
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = or <vscale x 2 x i1> [[TMP32]], [[TMP18]]
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = or <vscale x 2 x i1> [[TMP19]], [[TMP20]]
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP21]])
-; UNMASKED-SVE2P1-NEXT: [[TMP24]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP25]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i1> [[TMP3]]
-; UNMASKED-SVE2P1-NEXT: [[TMP26]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP18]], <vscale x 2 x i1> [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP33]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP20]], <vscale x 2 x i1> [[TMP5]]
-; UNMASKED-SVE2P1-NEXT: [[TMP27]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[VEC_PHI]]
-; UNMASKED-SVE2P1-NEXT: [[TMP28]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[VEC_PHI1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP29]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP30]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI3]]
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define i64 @find_last_i64_scaled_load(
+; CHECK-SAME: i64 [[N:%.*]], ptr noalias [[DATA:%.*]], i64 [[THRESHOLD:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[THRESHOLD]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 8 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 0)
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 2)
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 6)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP8]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
+; CHECK-NEXT: [[TMP15:%.*]] = freeze <vscale x 2 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP16:%.*]] = freeze <vscale x 2 x i1> [[TMP12]]
+; CHECK-NEXT: [[TMP18:%.*]] = freeze <vscale x 2 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP20:%.*]] = freeze <vscale x 2 x i1> [[TMP14]]
+; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 2 x i1> [[TMP15]], [[TMP16]]
+; CHECK-NEXT: [[TMP19:%.*]] = or <vscale x 2 x i1> [[TMP32]], [[TMP18]]
+; CHECK-NEXT: [[TMP21:%.*]] = or <vscale x 2 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP21]])
+; CHECK-NEXT: [[TMP24]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP25]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP26]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP18]], <vscale x 2 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP33]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP20]], <vscale x 2 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP27]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP28]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP29]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI2]]
+; CHECK-NEXT: [[TMP30]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI3]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
@@ -359,51 +370,51 @@ exit:
; Negative test: We should not use a wide load when the result of the load is extended.
; In this case, it's better to use SVE extending loads (rather than a multi-vector load).
define void @extending_load(ptr noalias %dst, ptr %src, i64 %n) {
-; UNMASKED-SVE2P1-LABEL: define void @extending_load(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = mul nuw nsw i64 [[TMP2]], 3
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP5]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; UNMASKED-SVE2P1-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD2]] to <vscale x 4 x i64>
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD3]] to <vscale x 4 x i64>
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD4]] to <vscale x 4 x i64>
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = mul nsw <vscale x 4 x i64> [[TMP9]], splat (i64 42)
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = mul nsw <vscale x 4 x i64> [[TMP10]], splat (i64 42)
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = mul nsw <vscale x 4 x i64> [[TMP11]], splat (i64 42)
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = mul nsw <vscale x 4 x i64> [[TMP12]], splat (i64 42)
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[DST]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP13]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP14]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP19]], <vscale x 4 x i64> [[TMP15]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP20]], <vscale x 4 x i64> [[TMP16]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x i64> [[TMP21]], ptr [[TMP17]], align 8
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define void @extending_load(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = mul nuw nsw i64 [[TMP2]], 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP3]], i64 [[TMP5]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD2]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD3]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP12:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD4]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <vscale x 4 x i64> [[TMP9]], splat (i64 42)
+; CHECK-NEXT: [[TMP14:%.*]] = mul nsw <vscale x 4 x i64> [[TMP10]], splat (i64 42)
+; CHECK-NEXT: [[TMP15:%.*]] = mul nsw <vscale x 4 x i64> [[TMP11]], splat (i64 42)
+; CHECK-NEXT: [[TMP16:%.*]] = mul nsw <vscale x 4 x i64> [[TMP12]], splat (i64 42)
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP13]], i64 0)
+; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP14]], i64 4)
+; CHECK-NEXT: [[TMP20:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP19]], <vscale x 4 x i64> [[TMP15]], i64 8)
+; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP20]], <vscale x 4 x i64> [[TMP16]], i64 12)
+; CHECK-NEXT: store <vscale x 16 x i64> [[TMP21]], ptr [[TMP17]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
@@ -426,70 +437,70 @@ exit:
; TODO: Support multi-vector operations with negative strides.
define void @reverse_stride(ptr noalias %c, ptr noalias %a, ptr noalias %b, i64 %n) {
-; UNMASKED-SVE2P1-LABEL: define void @reverse_stride(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[C:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; UNMASKED-SVE2P1-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[N]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SMIN]]
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 2
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP3]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = sub i64 [[N]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP6]]
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = sub nuw nsw i64 [[TMP4]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = sub i64 0, [[TMP8]]
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP9]]
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = sub i64 [[TMP9]], [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP27]]
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = mul i64 -2, [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = sub i64 [[TMP29]], [[TMP8]]
-; UNMASKED-SVE2P1-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP33]]
-; UNMASKED-SVE2P1-NEXT: [[TMP35:%.*]] = mul i64 -3, [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: [[TMP36:%.*]] = sub i64 [[TMP35]], [[TMP8]]
-; UNMASKED-SVE2P1-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP36]]
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = load <vscale x 4 x i32>, ptr [[TMP28]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP13:%.*]] = load <vscale x 4 x i32>, ptr [[TMP34]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = load <vscale x 4 x i32>, ptr [[TMP37]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP6]]
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP9]]
-; UNMASKED-SVE2P1-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP27]]
-; UNMASKED-SVE2P1-NEXT: [[TMP39:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP33]]
-; UNMASKED-SVE2P1-NEXT: [[TMP40:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP36]]
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = load <vscale x 4 x i32>, ptr [[TMP16]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = load <vscale x 4 x i32>, ptr [[TMP38]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = load <vscale x 4 x i32>, ptr [[TMP39]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = load <vscale x 4 x i32>, ptr [[TMP40]], align 4
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = add nsw <vscale x 4 x i32> [[TMP17]], [[TMP11]]
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = add nsw <vscale x 4 x i32> [[TMP18]], [[TMP12]]
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = add nsw <vscale x 4 x i32> [[TMP19]], [[TMP13]]
-; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = add nsw <vscale x 4 x i32> [[TMP20]], [[TMP14]]
-; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[TMP6]]
-; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP9]]
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP27]]
-; UNMASKED-SVE2P1-NEXT: [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP33]]
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP36]]
-; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP21]], ptr [[TMP26]], align 4
-; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP22]], ptr [[TMP30]], align 4
-; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP23]], ptr [[TMP41]], align 4
-; UNMASKED-SVE2P1-NEXT: store <vscale x 4 x i32> [[TMP24]], ptr [[TMP32]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define void @reverse_stride(
+; CHECK-SAME: ptr noalias [[C:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[N]], i64 0)
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SMIN]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = sub nuw nsw i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = sub i64 0, [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP27:%.*]] = sub i64 [[TMP9]], [[TMP4]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP27]]
+; CHECK-NEXT: [[TMP29:%.*]] = mul i64 -2, [[TMP4]]
+; CHECK-NEXT: [[TMP33:%.*]] = sub i64 [[TMP29]], [[TMP8]]
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP33]]
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 -3, [[TMP4]]
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 [[TMP35]], [[TMP8]]
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP11:%.*]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = load <vscale x 4 x i32>, ptr [[TMP28]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = load <vscale x 4 x i32>, ptr [[TMP34]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = load <vscale x 4 x i32>, ptr [[TMP37]], align 4
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP27]]
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP33]]
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 4 x i32>, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = load <vscale x 4 x i32>, ptr [[TMP38]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = load <vscale x 4 x i32>, ptr [[TMP39]], align 4
+; CHECK-NEXT: [[TMP20:%.*]] = load <vscale x 4 x i32>, ptr [[TMP40]], align 4
+; CHECK-NEXT: [[TMP21:%.*]] = add nsw <vscale x 4 x i32> [[TMP17]], [[TMP11]]
+; CHECK-NEXT: [[TMP22:%.*]] = add nsw <vscale x 4 x i32> [[TMP18]], [[TMP12]]
+; CHECK-NEXT: [[TMP23:%.*]] = add nsw <vscale x 4 x i32> [[TMP19]], [[TMP13]]
+; CHECK-NEXT: [[TMP24:%.*]] = add nsw <vscale x 4 x i32> [[TMP20]], [[TMP14]]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP27]]
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP33]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP36]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP21]], ptr [[TMP26]], align 4
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP22]], ptr [[TMP30]], align 4
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP23]], ptr [[TMP41]], align 4
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP24]], ptr [[TMP32]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %for.body
@@ -513,63 +524,63 @@ exit:
; Negative test (load): A strided load (stride = 2). Only the store should be widened to a VF multiple.
define void @gather_nxv4i32_stride2(ptr noalias %a, ptr noalias %b, i64 %n) {
-; UNMASKED-SVE2P1-LABEL: define void @gather_nxv4i32_stride2(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ITER_CHECK:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[TMP1]], i64 [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = mul i64 [[TMP2]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = shl i64 [[TMP2]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = mul i64 [[TMP8]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
-; UNMASKED-SVE2P1-NEXT: [[TMP12:%.*]] = mul i64 [[TMP2]], 3
-; UNMASKED-SVE2P1-NEXT: [[TMP14:%.*]] = mul i64 [[TMP12]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
-; UNMASKED-SVE2P1-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP17:%.*]] = shl i64 [[TMP7]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP18:%.*]] = shl i64 [[TMP11]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP19:%.*]] = shl i64 [[TMP15]], 1
-; UNMASKED-SVE2P1-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP16]]
-; UNMASKED-SVE2P1-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]]
-; UNMASKED-SVE2P1-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]]
-; UNMASKED-SVE2P1-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
-; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
-; UNMASKED-SVE2P1-NEXT: [[TMP24:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
-; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
-; UNMASKED-SVE2P1-NEXT: [[TMP25:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
-; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC4:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC5:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC4]])
-; UNMASKED-SVE2P1-NEXT: [[TMP26:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC5]], 0
-; UNMASKED-SVE2P1-NEXT: [[WIDE_VEC6:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
-; UNMASKED-SVE2P1-NEXT: [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC6]])
-; UNMASKED-SVE2P1-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC7]], 0
-; UNMASKED-SVE2P1-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; UNMASKED-SVE2P1-NEXT: [[TMP30:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP24]], i64 0)
-; UNMASKED-SVE2P1-NEXT: [[TMP31:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP30]], <vscale x 4 x float> [[TMP25]], i64 4)
-; UNMASKED-SVE2P1-NEXT: [[TMP32:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP31]], <vscale x 4 x float> [[TMP26]], i64 8)
-; UNMASKED-SVE2P1-NEXT: [[TMP29:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP32]], <vscale x 4 x float> [[TMP27]], i64 12)
-; UNMASKED-SVE2P1-NEXT: store <vscale x 16 x float> [[TMP29]], ptr [[TMP28]], align 4
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; UNMASKED-SVE2P1-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define void @gather_nxv4i32_stride2(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[TMP1]], i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP8]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP2]], 3
+; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP12]], 1
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = shl i64 [[TMP7]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP11]], 1
+; CHECK-NEXT: [[TMP19:%.*]] = shl i64 [[TMP15]], 1
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP24:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; CHECK-NEXT: [[TMP25:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC4]])
+; CHECK-NEXT: [[TMP26:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT: [[WIDE_VEC6:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
+; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC6]])
+; CHECK-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC7]], 0
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP30:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP24]], i64 0)
+; CHECK-NEXT: [[TMP31:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP30]], <vscale x 4 x float> [[TMP25]], i64 4)
+; CHECK-NEXT: [[TMP32:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP31]], <vscale x 4 x float> [[TMP26]], i64 8)
+; CHECK-NEXT: [[TMP29:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP32]], <vscale x 4 x float> [[TMP27]], i64 12)
+; CHECK-NEXT: store <vscale x 16 x float> [[TMP29]], ptr [[TMP28]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %for.body
@@ -590,43 +601,43 @@ exit:
}
define void @store_address(ptr noalias %x, ptr noalias %y, i64 %n) {
-; UNMASKED-SVE2P1-LABEL: define void @store_address(
-; UNMASKED-SVE2P1-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; UNMASKED-SVE2P1-NEXT: [[ENTRY:.*:]]
-; UNMASKED-SVE2P1-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; UNMASKED-SVE2P1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; UNMASKED-SVE2P1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; UNMASKED-SVE2P1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_PH]]:
-; UNMASKED-SVE2P1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 1
-; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
-; UNMASKED-SVE2P1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; UNMASKED-SVE2P1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
-; UNMASKED-SVE2P1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; UNMASKED-SVE2P1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; UNMASKED-SVE2P1-NEXT: br label %[[VECTOR_BODY:.*]]
-; UNMASKED-SVE2P1: [[VECTOR_BODY]]:
-; UNMASKED-SVE2P1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG26:![0-9]+]]
-; UNMASKED-SVE2P1-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG26]]
-; UNMASKED-SVE2P1-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
-; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_2:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
-; UNMASKED-SVE2P1-NEXT: [[STEP_ADD_3:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[VEC_IND]], !dbg [[DBG30:![0-9]+]]
-; UNMASKED-SVE2P1-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x ptr> [[WIDE_GEP]], i64 0
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD]], !dbg [[DBG30]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_2]], !dbg [[DBG30]]
-; UNMASKED-SVE2P1-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_3]], !dbg [[DBG30]]
-; UNMASKED-SVE2P1-NEXT: [[TMP6:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> poison, <vscale x 2 x ptr> [[WIDE_GEP]], i64 0), !dbg [[DBG31:![0-9]+]]
-; UNMASKED-SVE2P1-NEXT: [[TMP7:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP6]], <vscale x 2 x ptr> [[WIDE_GEP1]], i64 2), !dbg [[DBG31]]
-; UNMASKED-SVE2P1-NEXT: [[TMP8:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP7]], <vscale x 2 x ptr> [[WIDE_GEP2]], i64 4), !dbg [[DBG31]]
-; UNMASKED-SVE2P1-NEXT: [[TMP9:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP8]], <vscale x 2 x ptr> [[WIDE_GEP3]], i64 6), !dbg [[DBG31]]
-; UNMASKED-SVE2P1-NEXT: store <vscale x 8 x ptr> [[TMP9]], ptr [[TMP5]], align 8, !dbg [[DBG31]]
-; UNMASKED-SVE2P1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]], !dbg [[DBG26]]
-; UNMASKED-SVE2P1-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]], !dbg [[DBG26]]
-; UNMASKED-SVE2P1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG32:![0-9]+]]
-; UNMASKED-SVE2P1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG32]], !llvm.loop [[LOOP33:![0-9]+]]
-; UNMASKED-SVE2P1: [[MIDDLE_BLOCK]]:
+; CHECK-LABEL: define void @store_address(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG31:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG31]]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[VEC_IND]], !dbg [[DBG35:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x ptr> [[WIDE_GEP]], i64 0
+; CHECK-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD]], !dbg [[DBG35]]
+; CHECK-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_2]], !dbg [[DBG35]]
+; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i64, ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_3]], !dbg [[DBG35]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> poison, <vscale x 2 x ptr> [[WIDE_GEP]], i64 0), !dbg [[DBG36:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP6]], <vscale x 2 x ptr> [[WIDE_GEP1]], i64 2), !dbg [[DBG36]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP7]], <vscale x 2 x ptr> [[WIDE_GEP2]], i64 4), !dbg [[DBG36]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP8]], <vscale x 2 x ptr> [[WIDE_GEP3]], i64 6), !dbg [[DBG36]]
+; CHECK-NEXT: store <vscale x 8 x ptr> [[TMP9]], ptr [[TMP5]], align 8, !dbg [[DBG36]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]], !dbg [[DBG31]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG37:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG37]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
More information about the llvm-commits
mailing list