[llvm] d445267 - [AArch64] Increase the max interleave factor to 4 for loops with reductions (#205612)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 07:54:48 PDT 2026
Author: David Green
Date: 2026-06-26T14:54:41Z
New Revision: d445267fbb569e67ba9311f8c2bc675fd79e5d16
URL: https://github.com/llvm/llvm-project/commit/d445267fbb569e67ba9311f8c2bc675fd79e5d16
DIFF: https://github.com/llvm/llvm-project/commit/d445267fbb569e67ba9311f8c2bc675fd79e5d16.diff
LOG: [AArch64] Increase the max interleave factor to 4 for loops with reductions (#205612)
The default max interleave factor is 2. Increasing it to 4 universally
can spend an amount of codeside on something that does not always
increase performance (especially if the loop gets over-unrolled). Small
reduction loops often benefit from extra interleaving due to the
multiple independant streams that can execute in parallel. This patch
increases the max interleave factor to 4 for such loops, limited to
where the VF is <= 4 to limit the impact for already highly vectorized
loops.
Added:
Modified:
llvm/include/llvm/Analysis/TargetTransformInfo.h
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/include/llvm/CodeGen/BasicTTIImpl.h
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
llvm/lib/Target/ARM/ARMTargetTransformInfo.h
llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
llvm/lib/Target/X86/X86TargetTransformInfo.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
llvm/test/Transforms/LoopVectorize/AArch64/clamped-load.ll
llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll
llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
llvm/test/Transforms/PhaseOrdering/AArch64/peel-multiple-unreachable-exits-for-vectorization.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index aaf5ed6de93b6..10c0509460b95 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1498,8 +1498,11 @@ class TargetTransformInfo {
/// \return The maximum interleave factor that any transform should try to
/// perform for this target. This number depends on the level of parallelism
- /// and the number of execution units in the CPU.
- LLVM_ABI unsigned getMaxInterleaveFactor(ElementCount VF) const;
+ /// and the number of execution units in the CPU. HasUnorderedReductions
+ /// specifies whether (unordered) reductions are present in the loop being
+ /// vectorized.
+ LLVM_ABI unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const;
/// Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
LLVM_ABI static OperandValueInfo getOperandInfo(const Value *V);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 3db7729df2476..5645cc63a6944 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -722,7 +722,10 @@ class LLVM_ABI TargetTransformInfoImplBase {
return InstructionCost::getInvalid();
}
- virtual unsigned getMaxInterleaveFactor(ElementCount VF) const { return 1; }
+ virtual unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
+ return 1;
+ }
virtual InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
@@ -1170,7 +1173,7 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual bool preferEpilogueVectorization(ElementCount Iters) const {
// We consider epilogue vectorization unprofitable for targets that
// don't consider interleaving beneficial (eg. MVE).
- return getMaxInterleaveFactor(Iters) > 1;
+ return getMaxInterleaveFactor(Iters, false) > 1;
}
virtual bool shouldConsiderVectorizationRegPressure() const { return false; }
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 8c4df6664cf1d..55163eb1976bf 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1050,7 +1050,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
}
}
- unsigned getMaxInterleaveFactor(ElementCount VF) const override { return 1; }
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override {
+ return 1;
+ }
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 8e0c30f26e975..cdccd04f4c9ea 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -926,8 +926,10 @@ InstructionCost TargetTransformInfo::getPartialReductionCost(
BinOp, CostKind, FMF);
}
-unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
- return TTIImpl->getMaxInterleaveFactor(VF);
+unsigned
+TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
+ return TTIImpl->getMaxInterleaveFactor(VF, HasUnorderedReductions);
}
TargetTransformInfo::OperandValueInfo
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index bba685e4b4e0b..d69e8be940980 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5288,8 +5288,10 @@ bool AArch64TTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
(ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2());
}
-unsigned AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
- if (VF.isScalar())
+unsigned
+AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
+ if (VF.isScalar() || (HasUnorderedReductions && VF.getKnownMinValue() <= 4))
return 4;
return ST->getMaxInterleaveFactor();
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 5df5f22793cc8..24934cb01cdd1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -171,7 +171,8 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
return VF.getKnownMinValue() * ST->getVScaleForTuning();
}
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
bool prefersVectorizedAddressing() const override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 40227c496a589..9fe7544f00f15 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -488,7 +488,8 @@ void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
}
}
-unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
// Disable unrolling if the loop is not vectorized.
// TODO: Enable this again.
if (VF.isScalar())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 4a239f4d6983d..e20ef3f7524a6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -160,7 +160,8 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
Align SrcAlign, Align DestAlign,
std::optional<uint32_t> AtomicCpySize) const override;
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
bool getTgtMemIntrinsic(IntrinsicInst *Inst,
MemIntrinsicInfo &Info) const override;
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
index 1ab15b13d7698..688638b5067a5 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
@@ -83,7 +83,9 @@ bool R600TTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
}
-unsigned R600TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+unsigned
+R600TTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
// Disable unrolling if the loop is not vectorized.
// TODO: Enable this again.
if (VF.isScalar())
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
index f51c0963512c9..3afb2c35b1318 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
@@ -57,7 +57,8 @@ class R600TTIImpl final : public BasicTTIImplBase<R600TTIImpl> {
unsigned AddrSpace) const override;
bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
unsigned AddrSpace) const override;
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index e824839e39159..6eea0e64951f1 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -278,7 +278,8 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
llvm_unreachable("Unsupported register kind");
}
- unsigned getMaxInterleaveFactor(ElementCount VF) const override {
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override {
return ST->getMaxInterleaveFactor();
}
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 1d9d2856e16c6..8ff79f7f4f54d 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -115,7 +115,9 @@ unsigned HexagonTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
return 32;
}
-unsigned HexagonTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+unsigned
+HexagonTTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
return useHVX() ? 2 : 1;
}
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 86b3195222bdb..39517f8c64c88 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -83,7 +83,8 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
/// @{
unsigned getNumberOfRegisters(unsigned ClassID) const override;
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
unsigned getMinVectorRegisterBitWidth() const override;
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
index 5107c8def3799..9f63714db64a1 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
@@ -67,7 +67,9 @@ unsigned LoongArchTTIImpl::getRegisterClassForType(bool Vector,
return LoongArchRegisterClass::GPRRC;
}
-unsigned LoongArchTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+unsigned
+LoongArchTTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
return ST->getMaxInterleaveFactor();
}
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
index 88ecd8bc618f0..2392b709767ee 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
@@ -45,7 +45,8 @@ class LoongArchTTIImpl : public BasicTTIImplBase<LoongArchTTIImpl> {
unsigned getNumberOfRegisters(unsigned ClassID) const override;
unsigned getRegisterClassForType(bool Vector,
Type *Ty = nullptr) const override;
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
const char *getRegisterClassName(unsigned ClassID) const override;
TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override;
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 7f4bd170f8719..667195ea410ef 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -526,7 +526,8 @@ unsigned PPCTTIImpl::getPrefetchDistance() const {
return 300;
}
-unsigned PPCTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+unsigned PPCTTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
unsigned Directive = ST->getCPUDirective();
// The 440 has no SIMD support, but floating-point instructions
// have a 5-cycle latency, so unroll by 5x for latency hiding.
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 67ac457063d66..ce8eeab1fbc47 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -101,7 +101,8 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
unsigned getCacheLineSize() const override;
unsigned getPrefetchDistance() const override;
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
InstructionCost vectorCostAdjustmentFactor(unsigned Opcode, Type *Ty1,
Type *Ty2) const;
InstructionCost getArithmeticInstrCost(
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 2b43b93daa6c7..fc1f0c26c1a5e 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -450,7 +450,8 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
}
}
- unsigned getMaxInterleaveFactor(ElementCount VF) const override {
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override {
// Don't interleave if the loop has been vectorized with scalable vectors.
if (VF.isScalable())
return 1;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 8932b90f5ff02..5353c36fd9b97 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -478,7 +478,9 @@ unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
return ST->hasMiscellaneousExtensions3() ? 8192 : 2048;
}
-unsigned SystemZTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+unsigned
+SystemZTTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
return VF.isVector() ? 8 : 1;
}
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 8baa371cf47a7..06f39e585cd47 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -82,7 +82,8 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
bool HasCall) const override;
bool enableWritePrefetching() const override { return true; }
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
bool hasDivRemOp(Type *DataType, bool IsSigned) const override;
bool prefersVectorizedAddressing() const override { return false; }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 0dee9be6667c4..e139be77e4a44 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -233,7 +233,8 @@ unsigned X86TTIImpl::getLoadStoreVecRegBitWidth(unsigned) const {
.getFixedValue();
}
-unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const {
// If the loop will not be vectorized, don't interleave the loop.
// Let regular unroll to unroll the loop, which saves the overflow
// check and memory check cost.
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 556e997b01bd3..22171f5469d98 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -64,7 +64,8 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
unsigned getLoadStoreVecRegBitWidth(unsigned AS) const override;
- unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ unsigned getMaxInterleaveFactor(ElementCount VF,
+ bool HasUnorderedReductions) const override;
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 41112574d76e3..15b5d23489545 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1934,7 +1934,9 @@ static bool isIndvarOverflowCheckKnownFalse(
const LoopVectorizationCostModel *Cost,
ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
// Always be conservative if we don't know the exact unroll factor.
- unsigned MaxUF = UF ? *UF : Cost->TTI.getMaxInterleaveFactor(VF);
+ unsigned MaxUF = UF ? *UF
+ : std::max(Cost->TTI.getMaxInterleaveFactor(VF, false),
+ Cost->TTI.getMaxInterleaveFactor(VF, true));
IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
APInt MaxUIntTripCount = IdxTy->getMask();
@@ -3740,7 +3742,15 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
}
// Clamp the interleave ranges to reasonable counts.
- unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
+ bool HasUnorderedReductions =
+ HasReductions &&
+ !any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
+ [](VPRecipeBase &R) {
+ auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
+ return RedR && RedR->isOrdered();
+ });
+ unsigned MaxInterleaveCount =
+ TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
<< MaxInterleaveCount << "\n");
@@ -8374,7 +8384,8 @@ LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
// vector registers, loop vectorization may still enable scalar
// interleaving.
if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) &&
- TTI->getMaxInterleaveFactor(ElementCount::getFixed(1)) < 2)
+ (TTI->getMaxInterleaveFactor(ElementCount::getFixed(1), false) < 2 ||
+ TTI->getMaxInterleaveFactor(ElementCount::getFixed(1), true) < 2))
return LoopVectorizeResult(false, false);
bool Changed = false, CFGChanged = false;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
index b479b4d204360..3eaf4c4b1b3f4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
@@ -151,33 +151,43 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[CMP0:%.*]] = icmp eq i64 [[N]], 0
; A320-NEXT: br i1 [[CMP0]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
; A320: [[LOOP_PREHEADER]]:
-; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
; A320-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[VEC_PHI:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[VEC_PHI1:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI2:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI3:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP1]]
; A320-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 2
+; A320-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 4
+; A320-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 6
; A320-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
; A320-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP6]], align 8
+; A320-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; A320-NEXT: [[WIDE_LOAD6:%.*]] = load <2 x double>, ptr [[TMP11]], align 8
; A320-NEXT: [[TMP2]] = fadd fast <2 x double> [[VEC_PHI]], [[WIDE_LOAD]]
; A320-NEXT: [[TMP4]] = fadd fast <2 x double> [[VEC_PHI1]], [[WIDE_LOAD2]]
-; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 4
+; A320-NEXT: [[TMP10]] = fadd fast <2 x double> [[VEC_PHI2]], [[WIDE_LOAD5]]
+; A320-NEXT: [[TMP7]] = fadd fast <2 x double> [[VEC_PHI3]], [[WIDE_LOAD6]]
+; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 8
; A320-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; A320-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP4]], [[TMP2]]
-; A320-NEXT: [[TMP5:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX]])
+; A320-NEXT: [[BIN_RDX7:%.*]] = fadd fast <2 x double> [[TMP10]], [[BIN_RDX]]
+; A320-NEXT: [[BIN_RDX8:%.*]] = fadd fast <2 x double> [[TMP7]], [[BIN_RDX7]]
+; A320-NEXT: [[TMP9:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX8]])
; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; A320: [[SCALAR_PH]]:
; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
-; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ]
+; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ]
; A320-NEXT: br label %[[LOOP:.*]]
; A320: [[LOOP]]:
; A320-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -189,7 +199,7 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; A320: [[EXIT_LOOPEXIT]]:
-; A320-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi double [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
+; A320-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi double [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: br label %[[EXIT]]
; A320: [[EXIT]]:
; A320-NEXT: [[RES:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -232,39 +242,55 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[CMP0:%.*]] = icmp eq i64 [[N]], 0
; A320-NEXT: br i1 [[CMP0]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
; A320: [[LOOP_PREHEADER]]:
-; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
; A320-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[VEC_PHI:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[VEC_PHI1:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI2:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI3:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP1]]
; A320-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP1]]
; A320-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 2
+; A320-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 4
+; A320-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 6
; A320-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
; A320-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
+; A320-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x double>, ptr [[TMP9]], align 8
+; A320-NEXT: [[WIDE_LOAD6:%.*]] = load <2 x double>, ptr [[TMP12]], align 8
; A320-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP5]], i64 2
+; A320-NEXT: [[TMP13:%.*]] = getelementptr inbounds double, ptr [[TMP5]], i64 4
+; A320-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP5]], i64 6
; A320-NEXT: [[WIDE_LOAD3:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
; A320-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x double>, ptr [[TMP8]], align 8
+; A320-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x double>, ptr [[TMP13]], align 8
+; A320-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x double>, ptr [[TMP18]], align 8
; A320-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[WIDE_LOAD]], [[WIDE_LOAD3]]
; A320-NEXT: [[TMP10:%.*]] = fmul fast <2 x double> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
+; A320-NEXT: [[TMP19:%.*]] = fmul fast <2 x double> [[WIDE_LOAD5]], [[WIDE_LOAD9]]
+; A320-NEXT: [[TMP11:%.*]] = fmul fast <2 x double> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; A320-NEXT: [[TMP6]] = fadd fast <2 x double> [[VEC_PHI]], [[TMP4]]
; A320-NEXT: [[TMP7]] = fadd fast <2 x double> [[VEC_PHI1]], [[TMP10]]
-; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 4
+; A320-NEXT: [[TMP16]] = fadd fast <2 x double> [[VEC_PHI2]], [[TMP19]]
+; A320-NEXT: [[TMP15]] = fadd fast <2 x double> [[VEC_PHI3]], [[TMP11]]
+; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 8
; A320-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; A320-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP7]], [[TMP6]]
-; A320-NEXT: [[TMP9:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX]])
+; A320-NEXT: [[BIN_RDX11:%.*]] = fadd fast <2 x double> [[TMP16]], [[BIN_RDX]]
+; A320-NEXT: [[BIN_RDX12:%.*]] = fadd fast <2 x double> [[TMP15]], [[BIN_RDX11]]
+; A320-NEXT: [[TMP17:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX12]])
; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; A320: [[SCALAR_PH]]:
; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
-; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ]
+; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ]
; A320-NEXT: br label %[[LOOP:.*]]
; A320: [[LOOP]]:
; A320-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -279,7 +305,7 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; A320: [[EXIT_LOOPEXIT]]:
-; A320-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi double [ [[ACC_NEXT]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; A320-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi double [ [[ACC_NEXT]], %[[LOOP]] ], [ [[TMP17]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: br label %[[EXIT]]
; A320: [[EXIT]]:
; A320-NEXT: [[RES:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[ACC_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-load.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-load.ll
index 982949aa34ce2..d59240a6a8570 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-load.ll
@@ -7,7 +7,7 @@ define i32 @clamped_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK-LABEL: define i32 @clamped_load_reduction_bound2(
; CHECK-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[N]], 4
; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
@@ -16,47 +16,88 @@ define i32 @clamped_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP1]], [[TMP2]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -3
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -7
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -11
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -15
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD2]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD6]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[REVERSE10:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD7]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP11]] = add <4 x i32> [[VEC_PHI]], [[REVERSE]]
; CHECK-NEXT: [[TMP9]] = add <4 x i32> [[VEC_PHI1]], [[REVERSE3]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP14]] = add <4 x i32> [[VEC_PHI3]], [[REVERSE9]]
+; CHECK-NEXT: [[TMP13]] = add <4 x i32> [[VEC_PHI4]], [[REVERSE10]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP9]], [[TMP11]]
-; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX11:%.*]] = add <4 x i32> [[TMP14]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX12:%.*]] = add <4 x i32> [[TMP13]], [[BIN_RDX11]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX12]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[N_MOD_VF13:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_VEC14:%.*]] = sub i32 [[N]], [[N_MOD_VF13]]
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI16:%.*]] = phi <4 x i32> [ [[TMP16]], %[[VEC_EPILOG_PH]] ], [ [[TMP20:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[CLAMPED:%.*]] = urem i32 [[IV]], 2
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[CLAMPED]]
-; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[GEP]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <4 x i32>, ptr [[TMP19]], align 4
+; CHECK-NEXT: [[REVERSE18:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD17]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP20]] = add <4 x i32> [[VEC_PHI16]], [[REVERSE18]]
+; CHECK-NEXT: [[INDEX_NEXT19]] = add nuw i32 [[IV]], 4
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT19]], [[N_VEC14]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP20]])
+; CHECK-NEXT: [[CMP_N20:%.*]] = icmp eq i32 [[N]], [[N_VEC14]]
+; CHECK-NEXT: br i1 [[CMP_N20]], label %[[EXIT]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX21:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX21]], %[[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[CLAMPED1:%.*]] = urem i32 [[IV1]], 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[CLAMPED1]]
+; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP1]], align 4
; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[LV]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV1]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP1]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[R]]
;
entry:
@@ -82,51 +123,88 @@ define i32 @clamped_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK-LABEL: define i32 @clamped_load_reduction_bound4(
; CHECK-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i32 [[N]], 4
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[TMP0]], 3
; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 12
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP9]] = add <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP8]] = add <4 x i32> [[VEC_PHI2]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP11]] = add <4 x i32> [[VEC_PHI3]], [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP10]] = add <4 x i32> [[VEC_PHI4]], [[WIDE_LOAD7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX8:%.*]] = add <4 x i32> [[TMP11]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX9:%.*]] = add <4 x i32> [[TMP10]], [[BIN_RDX8]]
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX9]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_VEC11:%.*]] = sub i32 [[N]], [[N_MOD_VF10]]
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <4 x i32> [ [[TMP13]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[CLAMPED:%.*]] = urem i32 [[IV]], 4
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[CLAMPED]]
-; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP16]] = add <4 x i32> [[VEC_PHI13]], [[WIDE_LOAD14]]
+; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i32 [[IV]], 4
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT15]], [[N_VEC11]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP16]])
+; CHECK-NEXT: [[CMP_N16:%.*]] = icmp eq i32 [[N]], [[N_VEC11]]
+; CHECK-NEXT: br i1 [[CMP_N16]], label %[[EXIT]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX17]], %[[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[CLAMPED1:%.*]] = urem i32 [[IV1]], 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[CLAMPED1]]
+; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP1]], align 4
; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[LV]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV1]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP1]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[R]]
;
entry:
@@ -178,7 +256,7 @@ define i16 @clamped_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: [[TMP12]] = add <8 x i16> [[VEC_PHI2]], [[WIDE_LOAD3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <8 x i16> [[TMP12]], [[TMP5]]
; CHECK-NEXT: [[TMP14:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[BIN_RDX]])
@@ -186,7 +264,7 @@ define i16 @clamped_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: br i1 [[CMP_N1]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX1:%.*]] = phi i16 [ [[TMP14]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -203,7 +281,7 @@ define i16 @clamped_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: [[TMP6]] = add <4 x i16> [[VEC_PHI1]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[INDEX_NEXT9]] = add nuw i32 [[INDEX6]], 4
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT9]], [[N_VEC5]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP8:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP6]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC5]]
@@ -221,7 +299,7 @@ define i16 @clamped_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: [[SUM_NEXT]] = add i16 [[SUM]], [[LV]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[R:%.*]] = phi i16 [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ [[TMP8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[R]]
@@ -296,7 +374,7 @@ define i32 @clamped_user_ic_capped(ptr %A, i32 %N) {
; CHECK-NEXT: [[TMP18]] = add <4 x i32> [[VEC_PHI7]], [[WIDE_LOAD14]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP12]], [[TMP11]]
; CHECK-NEXT: [[BIN_RDX15:%.*]] = add <4 x i32> [[TMP13]], [[BIN_RDX]]
@@ -321,7 +399,7 @@ define i32 @clamped_user_ic_capped(ptr %A, i32 %N) {
; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[LV]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP20]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[SUM_NEXT_LCSSA]]
@@ -349,7 +427,7 @@ define i32 @scalable_vf_rejected(ptr %A, i64 %N) #0 {
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
@@ -358,7 +436,7 @@ define i32 @scalable_vf_rejected(ptr %A, i64 %N) #0 {
; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -366,24 +444,36 @@ define i32 @scalable_vf_rejected(ptr %A, i64 %N) #0 {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = urem i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP4]], 3
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP9]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4
; CHECK-NEXT: [[TMP11]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP12]] = add <vscale x 4 x i32> [[VEC_PHI1]], [[WIDE_MASKED_GATHER2]]
+; CHECK-NEXT: [[TMP15]] = add <vscale x 4 x i32> [[VEC_PHI2]], [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP16]] = add <vscale x 4 x i32> [[VEC_PHI3]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[TMP11]]
-; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX7:%.*]] = add <vscale x 4 x i32> [[TMP15]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX8:%.*]] = add <vscale x 4 x i32> [[TMP16]], [[BIN_RDX7]]
+; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -394,9 +484,9 @@ define i32 @scalable_vf_rejected(ptr %A, i64 %N) #0 {
; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[LV]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[R]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
index f7bc9591a66c6..a0ad85325a5d7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
@@ -473,35 +473,47 @@ define i32 @cse_multiple_ivs_with_scalar_resume(ptr %src, i64 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK: vector.main.loop.iter.check:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[SRC:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i64 4
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i16, ptr [[TMP1]], i64 8
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i16, ptr [[TMP1]], i64 12
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i16>, ptr [[TMP17]], align 2
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i16>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = sext <4 x i16> [[WIDE_LOAD3]] to <4 x i32>
+; CHECK-NEXT: [[TMP19:%.*]] = sext <4 x i16> [[WIDE_LOAD6]] to <4 x i32>
+; CHECK-NEXT: [[TMP20:%.*]] = sext <4 x i16> [[WIDE_LOAD7]] to <4 x i32>
; CHECK-NEXT: [[TMP5]] = or <4 x i32> [[TMP3]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP6]] = or <4 x i32> [[TMP4]], [[VEC_PHI2]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP15]] = or <4 x i32> [[TMP19]], [[VEC_PHI3]]
+; CHECK-NEXT: [[TMP16]] = or <4 x i32> [[TMP20]], [[VEC_PHI4]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i32> [[TMP6]], [[TMP5]]
-; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX8:%.*]] = or <4 x i32> [[TMP15]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX9:%.*]] = or <4 x i32> [[TMP16]], [[BIN_RDX8]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX9]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF19:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index a04367f32dd01..91ec377dc7db7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -42,42 +42,57 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fmaxnum(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 4
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 8
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 12
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP20]], align 4
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
-; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
+; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
+; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -89,7 +104,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -114,63 +129,93 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-LABEL: define float @test_fmax_and_fmin(
; CHECK-SAME: ptr [[SRC_0:%.*]], ptr [[SRC_1:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_0]], i64 [[IV]]
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_1]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_0]], i64 4
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_0]], i64 8
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_0]], i64 12
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC_0]], align 4
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x float>, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x float>, ptr [[TMP13]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_1]], i64 4
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_1]], i64 8
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_1]], i64 12
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[GEP_SRC_1]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP22]], align 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP28]], align 4
; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP10]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI6]], <4 x float> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP11]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI7]], <4 x float> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD5]])
; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD6]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP14]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[WIDE_LOAD13]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI5]], <4 x float> [[WIDE_LOAD14]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP29:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD13]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP34:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD14]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP9]]
; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
+; CHECK-NEXT: [[TMP35:%.*]] = freeze <4 x i1> [[TMP29]]
+; CHECK-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP18]], [[TMP35]]
+; CHECK-NEXT: [[TMP41:%.*]] = freeze <4 x i1> [[TMP34]]
+; CHECK-NEXT: [[TMP42:%.*]] = or <4 x i1> [[TMP38]], [[TMP41]]
+; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP42]])
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP21]]
; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]]
+; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI5]], <4 x float> [[TMP15]]
; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI6]], <4 x float> [[TMP10]]
+; CHECK-NEXT: [[TMP37:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI7]], <4 x float> [[TMP11]]
; CHECK-NEXT: [[TMP27:%.*]] = select i1 [[TMP19]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
-; CHECK-NEXT: [[TMP28:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP32]])
+; CHECK-NEXT: [[RDX_MINMAX16:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX15]], <4 x float> [[TMP33]])
+; CHECK-NEXT: [[TMP39:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX16]])
; CHECK-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP25]], <4 x float> [[TMP26]])
-; CHECK-NEXT: [[TMP29:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX9]])
+; CHECK-NEXT: [[RDX_MINMAX18:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX9]], <4 x float> [[TMP36]])
+; CHECK-NEXT: [[RDX_MINMAX19:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX18]], <4 x float> [[TMP37]])
+; CHECK-NEXT: [[TMP40:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX19]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP19]], true
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP29]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP39]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX20:%.*]] = phi float [ [[TMP40]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX20]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_0]], i64 [[IV1]]
; CHECK-NEXT: [[GEP_SRC_3:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_1]], i64 [[IV1]]
; CHECK-NEXT: [[L_0:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
@@ -181,8 +226,8 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP29]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[MIN_NEXT_LCSSA:%.*]] = phi float [ [[MIN_NEXT]], %[[LOOP]] ], [ [[TMP28]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP40]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MIN_NEXT_LCSSA:%.*]] = phi float [ [[MIN_NEXT]], %[[LOOP]] ], [ [[TMP39]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[SUB:%.*]] = fsub float [[MAX_NEXT_LCSSA]], [[MIN_NEXT_LCSSA]]
; CHECK-NEXT: ret float [[SUB]]
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index 0bddc498f9e83..fb2af1dbdf925 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -42,42 +42,57 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fminnum(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 4
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 8
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 12
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP20]], align 4
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
-; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
+; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
+; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -89,7 +104,7 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
index 9a9e55f2ec2d3..e2ab17e6cfe8f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
@@ -15,7 +15,7 @@ define i32 @cost_prefers_fixed_width_vf_but_force_scalable_vf(ptr noalias %dst,
; CHECK: Cost for VF vscale x 2: 9 (Estimated cost per lane: 4.5)
; CHECK: Cost for VF vscale x 4: 8 (Estimated cost per lane: 2)
; CHECK: LV: Selecting VF: vscale x 4.
-; CHECK: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={2}' {
+; CHECK: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={4}' {
;
entry:
br label %loop
@@ -43,7 +43,7 @@ define i32 @no_sve_fallback_to_neon(ptr %src, i64 %n) "target-features"="+neon"
; CHECK: Cost for VF 2: 4 (Estimated cost per lane: 2)
; CHECK: Cost for VF 4: 4 (Estimated cost per lane: 1)
; CHECK: LV: Selecting VF: 4.
-; CHECK: VPlan 'Final VPlan for VF={2,4},UF={2}' {
+; CHECK: VPlan 'Final VPlan for VF={2,4},UF={4}' {
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll
index f63e506f3adb1..5be10c24a611e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll
@@ -14,20 +14,29 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-LABEL: define float @_Z4testmm(
; CHECK-SAME: i64 [[SIZE:%.*]], i64 [[OFFSET:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SIZE]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SIZE]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SIZE]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[SIZE]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SIZE]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SIZE]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP64:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP65:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP106:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP107:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP148:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP149:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP64:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP65:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP151:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI9:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP152:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP106:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP107:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP213:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP214:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP148:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP149:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI11:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP297:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI12:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP298:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 3
@@ -35,6 +44,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP48:%.*]] = add i64 [[INDEX]], 5
; CHECK-NEXT: [[TMP155:%.*]] = add i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT: [[TMP153:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP163:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT: [[TMP164:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT: [[TMP165:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT: [[TMP166:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT: [[TMP167:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT: [[TMP168:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT: [[TMP169:%.*]] = add i64 [[INDEX]], 15
; CHECK-NEXT: [[ADD:%.*]] = add i64 [[INDEX]], [[OFFSET]]
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], [[OFFSET]]
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP16]], [[OFFSET]]
@@ -43,6 +60,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP48]], [[OFFSET]]
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP155]], [[OFFSET]]
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP7]], [[OFFSET]]
+; CHECK-NEXT: [[TMP170:%.*]] = add i64 [[TMP153]], [[OFFSET]]
+; CHECK-NEXT: [[TMP171:%.*]] = add i64 [[TMP163]], [[OFFSET]]
+; CHECK-NEXT: [[TMP172:%.*]] = add i64 [[TMP164]], [[OFFSET]]
+; CHECK-NEXT: [[TMP173:%.*]] = add i64 [[TMP165]], [[OFFSET]]
+; CHECK-NEXT: [[TMP174:%.*]] = add i64 [[TMP166]], [[OFFSET]]
+; CHECK-NEXT: [[TMP175:%.*]] = add i64 [[TMP167]], [[OFFSET]]
+; CHECK-NEXT: [[TMP176:%.*]] = add i64 [[TMP168]], [[OFFSET]]
+; CHECK-NEXT: [[TMP177:%.*]] = add i64 [[TMP169]], [[OFFSET]]
; CHECK-NEXT: [[MUL:%.*]] = mul i64 [[ADD]], 3
; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP9]], 3
; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP10]], 3
@@ -51,6 +76,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP13]], 3
; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP14]], 3
; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP15]], 3
+; CHECK-NEXT: [[TMP178:%.*]] = mul i64 [[TMP170]], 3
+; CHECK-NEXT: [[TMP195:%.*]] = mul i64 [[TMP171]], 3
+; CHECK-NEXT: [[TMP196:%.*]] = mul i64 [[TMP172]], 3
+; CHECK-NEXT: [[TMP199:%.*]] = mul i64 [[TMP173]], 3
+; CHECK-NEXT: [[TMP200:%.*]] = mul i64 [[TMP174]], 3
+; CHECK-NEXT: [[TMP203:%.*]] = mul i64 [[TMP175]], 3
+; CHECK-NEXT: [[TMP204:%.*]] = mul i64 [[TMP176]], 3
+; CHECK-NEXT: [[TMP207:%.*]] = mul i64 [[TMP177]], 3
; CHECK-NEXT: [[GEP_SRC_DATA:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[MUL]]
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP17]]
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP18]]
@@ -59,6 +92,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP21]]
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP22]]
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP23]]
+; CHECK-NEXT: [[TMP208:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP178]]
+; CHECK-NEXT: [[TMP211:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP195]]
+; CHECK-NEXT: [[TMP212:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP196]]
+; CHECK-NEXT: [[TMP215:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP199]]
+; CHECK-NEXT: [[TMP216:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP200]]
+; CHECK-NEXT: [[TMP217:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP203]]
+; CHECK-NEXT: [[TMP218:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP204]]
+; CHECK-NEXT: [[TMP219:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP207]]
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[GEP_SRC_DATA]], align 4
; CHECK-NEXT: [[TMP33:%.*]] = load float, ptr [[TMP25]], align 4
; CHECK-NEXT: [[TMP34:%.*]] = load float, ptr [[TMP26]], align 4
@@ -75,32 +116,74 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP41]], i32 1
; CHECK-NEXT: [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP42]], i32 2
; CHECK-NEXT: [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP43]], i32 3
+; CHECK-NEXT: [[TMP220:%.*]] = load float, ptr [[TMP208]], align 4
+; CHECK-NEXT: [[TMP221:%.*]] = load float, ptr [[TMP211]], align 4
+; CHECK-NEXT: [[TMP222:%.*]] = load float, ptr [[TMP212]], align 4
+; CHECK-NEXT: [[TMP231:%.*]] = load float, ptr [[TMP215]], align 4
+; CHECK-NEXT: [[TMP232:%.*]] = insertelement <4 x float> poison, float [[TMP220]], i32 0
+; CHECK-NEXT: [[TMP233:%.*]] = insertelement <4 x float> [[TMP232]], float [[TMP221]], i32 1
+; CHECK-NEXT: [[TMP234:%.*]] = insertelement <4 x float> [[TMP233]], float [[TMP222]], i32 2
+; CHECK-NEXT: [[TMP235:%.*]] = insertelement <4 x float> [[TMP234]], float [[TMP231]], i32 3
+; CHECK-NEXT: [[TMP236:%.*]] = load float, ptr [[TMP216]], align 4
+; CHECK-NEXT: [[TMP237:%.*]] = load float, ptr [[TMP217]], align 4
+; CHECK-NEXT: [[TMP238:%.*]] = load float, ptr [[TMP218]], align 4
+; CHECK-NEXT: [[TMP247:%.*]] = load float, ptr [[TMP219]], align 4
+; CHECK-NEXT: [[TMP248:%.*]] = insertelement <4 x float> poison, float [[TMP236]], i32 0
+; CHECK-NEXT: [[TMP249:%.*]] = insertelement <4 x float> [[TMP248]], float [[TMP237]], i32 1
+; CHECK-NEXT: [[TMP250:%.*]] = insertelement <4 x float> [[TMP249]], float [[TMP238]], i32 2
+; CHECK-NEXT: [[TMP251:%.*]] = insertelement <4 x float> [[TMP250]], float [[TMP247]], i32 3
; CHECK-NEXT: [[GEP_KERNEL:%.*]] = getelementptr inbounds [512 x float], ptr @kernel, i64 0, i64 [[INDEX]]
; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds float, ptr [[GEP_KERNEL]], i64 4
+; CHECK-NEXT: [[TMP252:%.*]] = getelementptr inbounds float, ptr [[GEP_KERNEL]], i64 8
+; CHECK-NEXT: [[TMP253:%.*]] = getelementptr inbounds float, ptr [[GEP_KERNEL]], i64 12
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_KERNEL]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP49]], align 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP252]], align 4
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x float>, ptr [[TMP253]], align 4
; CHECK-NEXT: [[TMP50:%.*]] = fmul fast <4 x float> [[TMP39]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP51:%.*]] = fmul fast <4 x float> [[TMP47]], [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP254:%.*]] = fmul fast <4 x float> [[TMP235]], [[WIDE_LOAD14]]
+; CHECK-NEXT: [[TMP255:%.*]] = fmul fast <4 x float> [[TMP251]], [[WIDE_LOAD15]]
; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds [512 x float], ptr @kernel2, i64 0, i64 [[INDEX]]
; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds float, ptr [[TMP52]], i64 4
+; CHECK-NEXT: [[TMP256:%.*]] = getelementptr inbounds float, ptr [[TMP52]], i64 8
+; CHECK-NEXT: [[TMP257:%.*]] = getelementptr inbounds float, ptr [[TMP52]], i64 12
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP52]], align 4
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP53]], align 4
+; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x float>, ptr [[TMP256]], align 4
+; CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x float>, ptr [[TMP257]], align 4
; CHECK-NEXT: [[TMP54:%.*]] = fmul fast <4 x float> [[TMP50]], [[WIDE_LOAD7]]
; CHECK-NEXT: [[TMP55:%.*]] = fmul fast <4 x float> [[TMP51]], [[WIDE_LOAD8]]
+; CHECK-NEXT: [[TMP258:%.*]] = fmul fast <4 x float> [[TMP254]], [[WIDE_LOAD18]]
+; CHECK-NEXT: [[TMP259:%.*]] = fmul fast <4 x float> [[TMP255]], [[WIDE_LOAD19]]
; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds [512 x float], ptr @kernel3, i64 0, i64 [[INDEX]]
; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds float, ptr [[TMP56]], i64 4
+; CHECK-NEXT: [[TMP260:%.*]] = getelementptr inbounds float, ptr [[TMP56]], i64 8
+; CHECK-NEXT: [[TMP261:%.*]] = getelementptr inbounds float, ptr [[TMP56]], i64 12
; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x float>, ptr [[TMP56]], align 4
; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x float>, ptr [[TMP57]], align 4
+; CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <4 x float>, ptr [[TMP260]], align 4
+; CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <4 x float>, ptr [[TMP261]], align 4
; CHECK-NEXT: [[TMP58:%.*]] = fmul fast <4 x float> [[TMP54]], [[WIDE_LOAD9]]
; CHECK-NEXT: [[TMP59:%.*]] = fmul fast <4 x float> [[TMP55]], [[WIDE_LOAD10]]
+; CHECK-NEXT: [[TMP262:%.*]] = fmul fast <4 x float> [[TMP258]], [[WIDE_LOAD22]]
+; CHECK-NEXT: [[TMP279:%.*]] = fmul fast <4 x float> [[TMP259]], [[WIDE_LOAD23]]
; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds [512 x float], ptr @kernel4, i64 0, i64 [[INDEX]]
; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds float, ptr [[TMP60]], i64 4
+; CHECK-NEXT: [[TMP280:%.*]] = getelementptr inbounds float, ptr [[TMP60]], i64 8
+; CHECK-NEXT: [[TMP283:%.*]] = getelementptr inbounds float, ptr [[TMP60]], i64 12
; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x float>, ptr [[TMP60]], align 4
; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP61]], align 4
+; CHECK-NEXT: [[WIDE_LOAD26:%.*]] = load <4 x float>, ptr [[TMP280]], align 4
+; CHECK-NEXT: [[WIDE_LOAD27:%.*]] = load <4 x float>, ptr [[TMP283]], align 4
; CHECK-NEXT: [[TMP62:%.*]] = fmul fast <4 x float> [[TMP58]], [[WIDE_LOAD11]]
; CHECK-NEXT: [[TMP63:%.*]] = fmul fast <4 x float> [[TMP59]], [[WIDE_LOAD12]]
+; CHECK-NEXT: [[TMP284:%.*]] = fmul fast <4 x float> [[TMP262]], [[WIDE_LOAD26]]
+; CHECK-NEXT: [[TMP287:%.*]] = fmul fast <4 x float> [[TMP279]], [[WIDE_LOAD27]]
; CHECK-NEXT: [[TMP64]] = fadd fast <4 x float> [[VEC_PHI]], [[TMP62]]
; CHECK-NEXT: [[TMP65]] = fadd fast <4 x float> [[VEC_PHI1]], [[TMP63]]
+; CHECK-NEXT: [[TMP151]] = fadd fast <4 x float> [[VEC_PHI6]], [[TMP284]]
+; CHECK-NEXT: [[TMP152]] = fadd fast <4 x float> [[VEC_PHI9]], [[TMP287]]
; CHECK-NEXT: [[TMP66:%.*]] = add i64 [[MUL]], 1
; CHECK-NEXT: [[TMP67:%.*]] = add i64 [[TMP17]], 1
; CHECK-NEXT: [[TMP68:%.*]] = add i64 [[TMP18]], 1
@@ -109,6 +192,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP71:%.*]] = add i64 [[TMP21]], 1
; CHECK-NEXT: [[TMP72:%.*]] = add i64 [[TMP22]], 1
; CHECK-NEXT: [[TMP73:%.*]] = add i64 [[TMP23]], 1
+; CHECK-NEXT: [[TMP288:%.*]] = add i64 [[TMP178]], 1
+; CHECK-NEXT: [[TMP291:%.*]] = add i64 [[TMP195]], 1
+; CHECK-NEXT: [[TMP292:%.*]] = add i64 [[TMP196]], 1
+; CHECK-NEXT: [[TMP295:%.*]] = add i64 [[TMP199]], 1
+; CHECK-NEXT: [[TMP296:%.*]] = add i64 [[TMP200]], 1
+; CHECK-NEXT: [[TMP299:%.*]] = add i64 [[TMP203]], 1
+; CHECK-NEXT: [[TMP309:%.*]] = add i64 [[TMP204]], 1
+; CHECK-NEXT: [[TMP313:%.*]] = add i64 [[TMP207]], 1
; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP66]]
; CHECK-NEXT: [[TMP75:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP67]]
; CHECK-NEXT: [[TMP76:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP68]]
@@ -117,6 +208,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP79:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP71]]
; CHECK-NEXT: [[TMP80:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP72]]
; CHECK-NEXT: [[TMP81:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP73]]
+; CHECK-NEXT: [[TMP317:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP288]]
+; CHECK-NEXT: [[TMP156:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP291]]
+; CHECK-NEXT: [[TMP157:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP292]]
+; CHECK-NEXT: [[TMP158:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP295]]
+; CHECK-NEXT: [[TMP159:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP296]]
+; CHECK-NEXT: [[TMP160:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP299]]
+; CHECK-NEXT: [[TMP161:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP309]]
+; CHECK-NEXT: [[TMP162:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP313]]
; CHECK-NEXT: [[TMP82:%.*]] = load float, ptr [[TMP74]], align 4
; CHECK-NEXT: [[TMP83:%.*]] = load float, ptr [[TMP75]], align 4
; CHECK-NEXT: [[TMP84:%.*]] = load float, ptr [[TMP76]], align 4
@@ -133,16 +232,42 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP95:%.*]] = insertelement <4 x float> [[TMP94]], float [[TMP91]], i32 1
; CHECK-NEXT: [[TMP96:%.*]] = insertelement <4 x float> [[TMP95]], float [[TMP92]], i32 2
; CHECK-NEXT: [[TMP97:%.*]] = insertelement <4 x float> [[TMP96]], float [[TMP93]], i32 3
+; CHECK-NEXT: [[TMP179:%.*]] = load float, ptr [[TMP317]], align 4
+; CHECK-NEXT: [[TMP180:%.*]] = load float, ptr [[TMP156]], align 4
+; CHECK-NEXT: [[TMP181:%.*]] = load float, ptr [[TMP157]], align 4
+; CHECK-NEXT: [[TMP182:%.*]] = load float, ptr [[TMP158]], align 4
+; CHECK-NEXT: [[TMP183:%.*]] = insertelement <4 x float> poison, float [[TMP179]], i32 0
+; CHECK-NEXT: [[TMP184:%.*]] = insertelement <4 x float> [[TMP183]], float [[TMP180]], i32 1
+; CHECK-NEXT: [[TMP185:%.*]] = insertelement <4 x float> [[TMP184]], float [[TMP181]], i32 2
+; CHECK-NEXT: [[TMP186:%.*]] = insertelement <4 x float> [[TMP185]], float [[TMP182]], i32 3
+; CHECK-NEXT: [[TMP187:%.*]] = load float, ptr [[TMP159]], align 4
+; CHECK-NEXT: [[TMP188:%.*]] = load float, ptr [[TMP160]], align 4
+; CHECK-NEXT: [[TMP189:%.*]] = load float, ptr [[TMP161]], align 4
+; CHECK-NEXT: [[TMP190:%.*]] = load float, ptr [[TMP162]], align 4
+; CHECK-NEXT: [[TMP191:%.*]] = insertelement <4 x float> poison, float [[TMP187]], i32 0
+; CHECK-NEXT: [[TMP192:%.*]] = insertelement <4 x float> [[TMP191]], float [[TMP188]], i32 1
+; CHECK-NEXT: [[TMP193:%.*]] = insertelement <4 x float> [[TMP192]], float [[TMP189]], i32 2
+; CHECK-NEXT: [[TMP194:%.*]] = insertelement <4 x float> [[TMP193]], float [[TMP190]], i32 3
; CHECK-NEXT: [[TMP98:%.*]] = fmul fast <4 x float> [[WIDE_LOAD]], [[TMP89]]
; CHECK-NEXT: [[TMP99:%.*]] = fmul fast <4 x float> [[WIDE_LOAD6]], [[TMP97]]
+; CHECK-NEXT: [[TMP197:%.*]] = fmul fast <4 x float> [[WIDE_LOAD14]], [[TMP186]]
+; CHECK-NEXT: [[TMP198:%.*]] = fmul fast <4 x float> [[WIDE_LOAD15]], [[TMP194]]
; CHECK-NEXT: [[TMP100:%.*]] = fmul fast <4 x float> [[WIDE_LOAD7]], [[TMP98]]
; CHECK-NEXT: [[TMP101:%.*]] = fmul fast <4 x float> [[WIDE_LOAD8]], [[TMP99]]
+; CHECK-NEXT: [[TMP201:%.*]] = fmul fast <4 x float> [[WIDE_LOAD18]], [[TMP197]]
+; CHECK-NEXT: [[TMP202:%.*]] = fmul fast <4 x float> [[WIDE_LOAD19]], [[TMP198]]
; CHECK-NEXT: [[TMP102:%.*]] = fmul fast <4 x float> [[WIDE_LOAD9]], [[TMP100]]
; CHECK-NEXT: [[TMP103:%.*]] = fmul fast <4 x float> [[WIDE_LOAD10]], [[TMP101]]
+; CHECK-NEXT: [[TMP205:%.*]] = fmul fast <4 x float> [[WIDE_LOAD22]], [[TMP201]]
+; CHECK-NEXT: [[TMP206:%.*]] = fmul fast <4 x float> [[WIDE_LOAD23]], [[TMP202]]
; CHECK-NEXT: [[TMP104:%.*]] = fmul fast <4 x float> [[WIDE_LOAD11]], [[TMP102]]
; CHECK-NEXT: [[TMP105:%.*]] = fmul fast <4 x float> [[WIDE_LOAD12]], [[TMP103]]
+; CHECK-NEXT: [[TMP209:%.*]] = fmul fast <4 x float> [[WIDE_LOAD26]], [[TMP205]]
+; CHECK-NEXT: [[TMP210:%.*]] = fmul fast <4 x float> [[WIDE_LOAD27]], [[TMP206]]
; CHECK-NEXT: [[TMP106]] = fadd fast <4 x float> [[VEC_PHI2]], [[TMP104]]
; CHECK-NEXT: [[TMP107]] = fadd fast <4 x float> [[VEC_PHI3]], [[TMP105]]
+; CHECK-NEXT: [[TMP213]] = fadd fast <4 x float> [[VEC_PHI7]], [[TMP209]]
+; CHECK-NEXT: [[TMP214]] = fadd fast <4 x float> [[VEC_PHI8]], [[TMP210]]
; CHECK-NEXT: [[TMP108:%.*]] = add i64 [[MUL]], 2
; CHECK-NEXT: [[TMP109:%.*]] = add i64 [[TMP17]], 2
; CHECK-NEXT: [[TMP110:%.*]] = add i64 [[TMP18]], 2
@@ -151,6 +276,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP113:%.*]] = add i64 [[TMP21]], 2
; CHECK-NEXT: [[TMP114:%.*]] = add i64 [[TMP22]], 2
; CHECK-NEXT: [[TMP115:%.*]] = add i64 [[TMP23]], 2
+; CHECK-NEXT: [[TMP223:%.*]] = add i64 [[TMP178]], 2
+; CHECK-NEXT: [[TMP224:%.*]] = add i64 [[TMP195]], 2
+; CHECK-NEXT: [[TMP225:%.*]] = add i64 [[TMP196]], 2
+; CHECK-NEXT: [[TMP226:%.*]] = add i64 [[TMP199]], 2
+; CHECK-NEXT: [[TMP227:%.*]] = add i64 [[TMP200]], 2
+; CHECK-NEXT: [[TMP228:%.*]] = add i64 [[TMP203]], 2
+; CHECK-NEXT: [[TMP229:%.*]] = add i64 [[TMP204]], 2
+; CHECK-NEXT: [[TMP230:%.*]] = add i64 [[TMP207]], 2
; CHECK-NEXT: [[TMP116:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP108]]
; CHECK-NEXT: [[TMP117:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP109]]
; CHECK-NEXT: [[TMP118:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP110]]
@@ -159,6 +292,14 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP121:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP113]]
; CHECK-NEXT: [[TMP122:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP114]]
; CHECK-NEXT: [[TMP123:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP115]]
+; CHECK-NEXT: [[TMP239:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP223]]
+; CHECK-NEXT: [[TMP240:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP224]]
+; CHECK-NEXT: [[TMP241:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP225]]
+; CHECK-NEXT: [[TMP242:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP226]]
+; CHECK-NEXT: [[TMP243:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP227]]
+; CHECK-NEXT: [[TMP244:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP228]]
+; CHECK-NEXT: [[TMP245:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP229]]
+; CHECK-NEXT: [[TMP246:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP230]]
; CHECK-NEXT: [[TMP124:%.*]] = load float, ptr [[TMP116]], align 4
; CHECK-NEXT: [[TMP125:%.*]] = load float, ptr [[TMP117]], align 4
; CHECK-NEXT: [[TMP126:%.*]] = load float, ptr [[TMP118]], align 4
@@ -175,57 +316,195 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[TMP137:%.*]] = insertelement <4 x float> [[TMP136]], float [[TMP133]], i32 1
; CHECK-NEXT: [[TMP138:%.*]] = insertelement <4 x float> [[TMP137]], float [[TMP134]], i32 2
; CHECK-NEXT: [[TMP139:%.*]] = insertelement <4 x float> [[TMP138]], float [[TMP135]], i32 3
+; CHECK-NEXT: [[TMP263:%.*]] = load float, ptr [[TMP239]], align 4
+; CHECK-NEXT: [[TMP264:%.*]] = load float, ptr [[TMP240]], align 4
+; CHECK-NEXT: [[TMP265:%.*]] = load float, ptr [[TMP241]], align 4
+; CHECK-NEXT: [[TMP266:%.*]] = load float, ptr [[TMP242]], align 4
+; CHECK-NEXT: [[TMP267:%.*]] = insertelement <4 x float> poison, float [[TMP263]], i32 0
+; CHECK-NEXT: [[TMP268:%.*]] = insertelement <4 x float> [[TMP267]], float [[TMP264]], i32 1
+; CHECK-NEXT: [[TMP269:%.*]] = insertelement <4 x float> [[TMP268]], float [[TMP265]], i32 2
+; CHECK-NEXT: [[TMP270:%.*]] = insertelement <4 x float> [[TMP269]], float [[TMP266]], i32 3
+; CHECK-NEXT: [[TMP271:%.*]] = load float, ptr [[TMP243]], align 4
+; CHECK-NEXT: [[TMP272:%.*]] = load float, ptr [[TMP244]], align 4
+; CHECK-NEXT: [[TMP273:%.*]] = load float, ptr [[TMP245]], align 4
+; CHECK-NEXT: [[TMP274:%.*]] = load float, ptr [[TMP246]], align 4
+; CHECK-NEXT: [[TMP275:%.*]] = insertelement <4 x float> poison, float [[TMP271]], i32 0
+; CHECK-NEXT: [[TMP276:%.*]] = insertelement <4 x float> [[TMP275]], float [[TMP272]], i32 1
+; CHECK-NEXT: [[TMP277:%.*]] = insertelement <4 x float> [[TMP276]], float [[TMP273]], i32 2
+; CHECK-NEXT: [[TMP278:%.*]] = insertelement <4 x float> [[TMP277]], float [[TMP274]], i32 3
; CHECK-NEXT: [[TMP140:%.*]] = fmul fast <4 x float> [[WIDE_LOAD]], [[TMP131]]
; CHECK-NEXT: [[TMP141:%.*]] = fmul fast <4 x float> [[WIDE_LOAD6]], [[TMP139]]
+; CHECK-NEXT: [[TMP281:%.*]] = fmul fast <4 x float> [[WIDE_LOAD14]], [[TMP270]]
+; CHECK-NEXT: [[TMP282:%.*]] = fmul fast <4 x float> [[WIDE_LOAD15]], [[TMP278]]
; CHECK-NEXT: [[TMP142:%.*]] = fmul fast <4 x float> [[WIDE_LOAD7]], [[TMP140]]
; CHECK-NEXT: [[TMP143:%.*]] = fmul fast <4 x float> [[WIDE_LOAD8]], [[TMP141]]
+; CHECK-NEXT: [[TMP285:%.*]] = fmul fast <4 x float> [[WIDE_LOAD18]], [[TMP281]]
+; CHECK-NEXT: [[TMP286:%.*]] = fmul fast <4 x float> [[WIDE_LOAD19]], [[TMP282]]
; CHECK-NEXT: [[TMP144:%.*]] = fmul fast <4 x float> [[WIDE_LOAD9]], [[TMP142]]
; CHECK-NEXT: [[TMP145:%.*]] = fmul fast <4 x float> [[WIDE_LOAD10]], [[TMP143]]
+; CHECK-NEXT: [[TMP289:%.*]] = fmul fast <4 x float> [[WIDE_LOAD22]], [[TMP285]]
+; CHECK-NEXT: [[TMP290:%.*]] = fmul fast <4 x float> [[WIDE_LOAD23]], [[TMP286]]
; CHECK-NEXT: [[TMP146:%.*]] = fmul fast <4 x float> [[WIDE_LOAD11]], [[TMP144]]
; CHECK-NEXT: [[TMP147:%.*]] = fmul fast <4 x float> [[WIDE_LOAD12]], [[TMP145]]
+; CHECK-NEXT: [[TMP293:%.*]] = fmul fast <4 x float> [[WIDE_LOAD26]], [[TMP289]]
+; CHECK-NEXT: [[TMP294:%.*]] = fmul fast <4 x float> [[WIDE_LOAD27]], [[TMP290]]
; CHECK-NEXT: [[TMP148]] = fadd fast <4 x float> [[VEC_PHI4]], [[TMP146]]
; CHECK-NEXT: [[TMP149]] = fadd fast <4 x float> [[VEC_PHI5]], [[TMP147]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP297]] = fadd fast <4 x float> [[VEC_PHI11]], [[TMP293]]
+; CHECK-NEXT: [[TMP298]] = fadd fast <4 x float> [[VEC_PHI12]], [[TMP294]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP150:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP150]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP65]], [[TMP64]]
-; CHECK-NEXT: [[TMP151:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX28:%.*]] = fadd fast <4 x float> [[TMP151]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX29:%.*]] = fadd fast <4 x float> [[TMP152]], [[BIN_RDX28]]
+; CHECK-NEXT: [[TMP300:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX29]])
; CHECK-NEXT: [[BIN_RDX13:%.*]] = fadd fast <4 x float> [[TMP107]], [[TMP106]]
-; CHECK-NEXT: [[TMP152:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX13]])
+; CHECK-NEXT: [[BIN_RDX31:%.*]] = fadd fast <4 x float> [[TMP213]], [[BIN_RDX13]]
+; CHECK-NEXT: [[BIN_RDX32:%.*]] = fadd fast <4 x float> [[TMP214]], [[BIN_RDX31]]
+; CHECK-NEXT: [[TMP301:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX32]])
; CHECK-NEXT: [[BIN_RDX14:%.*]] = fadd fast <4 x float> [[TMP149]], [[TMP148]]
-; CHECK-NEXT: [[TMP153:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX14]])
+; CHECK-NEXT: [[BIN_RDX34:%.*]] = fadd fast <4 x float> [[TMP297]], [[BIN_RDX14]]
+; CHECK-NEXT: [[BIN_RDX35:%.*]] = fadd fast <4 x float> [[TMP298]], [[BIN_RDX34]]
+; CHECK-NEXT: [[TMP302:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX35]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SIZE]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP151]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX15:%.*]] = phi float [ [[TMP152]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi float [ [[TMP153]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP300]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX36:%.*]] = phi float [ [[TMP301]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX37:%.*]] = phi float [ [[TMP302]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[N_MOD_VF38:%.*]] = urem i64 [[SIZE]], 4
+; CHECK-NEXT: [[N_VEC39:%.*]] = sub i64 [[SIZE]], [[N_MOD_VF38]]
+; CHECK-NEXT: [[TMP303:%.*]] = insertelement <4 x float> zeroinitializer, float [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[TMP304:%.*]] = insertelement <4 x float> zeroinitializer, float [[BC_MERGE_RDX36]], i32 0
+; CHECK-NEXT: [[TMP305:%.*]] = insertelement <4 x float> zeroinitializer, float [[BC_MERGE_RDX37]], i32 0
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RDX_0:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RDX_0_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RDX_1:%.*]] = phi float [ [[BC_MERGE_RDX15]], %[[SCALAR_PH]] ], [ [[RDX_1_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED_2:%.*]] = phi float [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[RDX_2_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT48:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI41:%.*]] = phi <4 x float> [ [[TMP303]], %[[VEC_EPILOG_PH]] ], [ [[TMP337:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI42:%.*]] = phi <4 x float> [ [[TMP304]], %[[VEC_EPILOG_PH]] ], [ [[TMP358:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI43:%.*]] = phi <4 x float> [ [[TMP305]], %[[VEC_EPILOG_PH]] ], [ [[TMP379:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP306:%.*]] = add i64 [[IV1]], 1
+; CHECK-NEXT: [[TMP307:%.*]] = add i64 [[IV1]], 2
+; CHECK-NEXT: [[TMP308:%.*]] = add i64 [[IV1]], 3
; CHECK-NEXT: [[ADD1:%.*]] = add i64 [[IV1]], [[OFFSET]]
+; CHECK-NEXT: [[TMP310:%.*]] = add i64 [[TMP306]], [[OFFSET]]
+; CHECK-NEXT: [[TMP311:%.*]] = add i64 [[TMP307]], [[OFFSET]]
+; CHECK-NEXT: [[TMP312:%.*]] = add i64 [[TMP308]], [[OFFSET]]
; CHECK-NEXT: [[MUL1:%.*]] = mul i64 [[ADD1]], 3
+; CHECK-NEXT: [[TMP314:%.*]] = mul i64 [[TMP310]], 3
+; CHECK-NEXT: [[TMP315:%.*]] = mul i64 [[TMP311]], 3
+; CHECK-NEXT: [[TMP316:%.*]] = mul i64 [[TMP312]], 3
; CHECK-NEXT: [[GEP_SRC_DATA1:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[MUL1]]
-; CHECK-NEXT: [[TMP154:%.*]] = load float, ptr [[GEP_SRC_DATA1]], align 4
+; CHECK-NEXT: [[TMP318:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP314]]
+; CHECK-NEXT: [[TMP319:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP315]]
+; CHECK-NEXT: [[TMP320:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP316]]
+; CHECK-NEXT: [[TMP321:%.*]] = load float, ptr [[GEP_SRC_DATA1]], align 4
+; CHECK-NEXT: [[TMP322:%.*]] = load float, ptr [[TMP318]], align 4
+; CHECK-NEXT: [[TMP323:%.*]] = load float, ptr [[TMP319]], align 4
+; CHECK-NEXT: [[TMP324:%.*]] = load float, ptr [[TMP320]], align 4
+; CHECK-NEXT: [[TMP325:%.*]] = insertelement <4 x float> poison, float [[TMP321]], i32 0
+; CHECK-NEXT: [[TMP326:%.*]] = insertelement <4 x float> [[TMP325]], float [[TMP322]], i32 1
+; CHECK-NEXT: [[TMP327:%.*]] = insertelement <4 x float> [[TMP326]], float [[TMP323]], i32 2
+; CHECK-NEXT: [[TMP328:%.*]] = insertelement <4 x float> [[TMP327]], float [[TMP324]], i32 3
; CHECK-NEXT: [[GEP_KERNEL1:%.*]] = getelementptr inbounds [512 x float], ptr @kernel, i64 0, i64 [[IV1]]
-; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[GEP_KERNEL1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD44:%.*]] = load <4 x float>, ptr [[GEP_KERNEL1]], align 4
+; CHECK-NEXT: [[TMP330:%.*]] = fmul fast <4 x float> [[TMP328]], [[WIDE_LOAD44]]
+; CHECK-NEXT: [[TMP331:%.*]] = getelementptr inbounds [512 x float], ptr @kernel2, i64 0, i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD45:%.*]] = load <4 x float>, ptr [[TMP331]], align 4
+; CHECK-NEXT: [[TMP332:%.*]] = fmul fast <4 x float> [[TMP330]], [[WIDE_LOAD45]]
+; CHECK-NEXT: [[TMP333:%.*]] = getelementptr inbounds [512 x float], ptr @kernel3, i64 0, i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD46:%.*]] = load <4 x float>, ptr [[TMP333]], align 4
+; CHECK-NEXT: [[TMP334:%.*]] = fmul fast <4 x float> [[TMP332]], [[WIDE_LOAD46]]
+; CHECK-NEXT: [[TMP335:%.*]] = getelementptr inbounds [512 x float], ptr @kernel4, i64 0, i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD47:%.*]] = load <4 x float>, ptr [[TMP335]], align 4
+; CHECK-NEXT: [[TMP336:%.*]] = fmul fast <4 x float> [[TMP334]], [[WIDE_LOAD47]]
+; CHECK-NEXT: [[TMP337]] = fadd fast <4 x float> [[VEC_PHI41]], [[TMP336]]
+; CHECK-NEXT: [[TMP338:%.*]] = add i64 [[MUL1]], 1
+; CHECK-NEXT: [[TMP339:%.*]] = add i64 [[TMP314]], 1
+; CHECK-NEXT: [[TMP340:%.*]] = add i64 [[TMP315]], 1
+; CHECK-NEXT: [[TMP341:%.*]] = add i64 [[TMP316]], 1
+; CHECK-NEXT: [[TMP342:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP338]]
+; CHECK-NEXT: [[TMP343:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP339]]
+; CHECK-NEXT: [[TMP344:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP340]]
+; CHECK-NEXT: [[TMP345:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP341]]
+; CHECK-NEXT: [[TMP346:%.*]] = load float, ptr [[TMP342]], align 4
+; CHECK-NEXT: [[TMP347:%.*]] = load float, ptr [[TMP343]], align 4
+; CHECK-NEXT: [[TMP348:%.*]] = load float, ptr [[TMP344]], align 4
+; CHECK-NEXT: [[TMP349:%.*]] = load float, ptr [[TMP345]], align 4
+; CHECK-NEXT: [[TMP350:%.*]] = insertelement <4 x float> poison, float [[TMP346]], i32 0
+; CHECK-NEXT: [[TMP351:%.*]] = insertelement <4 x float> [[TMP350]], float [[TMP347]], i32 1
+; CHECK-NEXT: [[TMP352:%.*]] = insertelement <4 x float> [[TMP351]], float [[TMP348]], i32 2
+; CHECK-NEXT: [[TMP353:%.*]] = insertelement <4 x float> [[TMP352]], float [[TMP349]], i32 3
+; CHECK-NEXT: [[TMP354:%.*]] = fmul fast <4 x float> [[WIDE_LOAD44]], [[TMP353]]
+; CHECK-NEXT: [[TMP355:%.*]] = fmul fast <4 x float> [[WIDE_LOAD45]], [[TMP354]]
+; CHECK-NEXT: [[TMP356:%.*]] = fmul fast <4 x float> [[WIDE_LOAD46]], [[TMP355]]
+; CHECK-NEXT: [[TMP357:%.*]] = fmul fast <4 x float> [[WIDE_LOAD47]], [[TMP356]]
+; CHECK-NEXT: [[TMP358]] = fadd fast <4 x float> [[VEC_PHI42]], [[TMP357]]
+; CHECK-NEXT: [[TMP359:%.*]] = add i64 [[MUL1]], 2
+; CHECK-NEXT: [[TMP360:%.*]] = add i64 [[TMP314]], 2
+; CHECK-NEXT: [[TMP361:%.*]] = add i64 [[TMP315]], 2
+; CHECK-NEXT: [[TMP362:%.*]] = add i64 [[TMP316]], 2
+; CHECK-NEXT: [[TMP363:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP359]]
+; CHECK-NEXT: [[TMP364:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP360]]
+; CHECK-NEXT: [[TMP365:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP361]]
+; CHECK-NEXT: [[TMP366:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[TMP362]]
+; CHECK-NEXT: [[TMP367:%.*]] = load float, ptr [[TMP363]], align 4
+; CHECK-NEXT: [[TMP368:%.*]] = load float, ptr [[TMP364]], align 4
+; CHECK-NEXT: [[TMP369:%.*]] = load float, ptr [[TMP365]], align 4
+; CHECK-NEXT: [[TMP370:%.*]] = load float, ptr [[TMP366]], align 4
+; CHECK-NEXT: [[TMP371:%.*]] = insertelement <4 x float> poison, float [[TMP367]], i32 0
+; CHECK-NEXT: [[TMP372:%.*]] = insertelement <4 x float> [[TMP371]], float [[TMP368]], i32 1
+; CHECK-NEXT: [[TMP373:%.*]] = insertelement <4 x float> [[TMP372]], float [[TMP369]], i32 2
+; CHECK-NEXT: [[TMP374:%.*]] = insertelement <4 x float> [[TMP373]], float [[TMP370]], i32 3
+; CHECK-NEXT: [[TMP375:%.*]] = fmul fast <4 x float> [[WIDE_LOAD44]], [[TMP374]]
+; CHECK-NEXT: [[TMP376:%.*]] = fmul fast <4 x float> [[WIDE_LOAD45]], [[TMP375]]
+; CHECK-NEXT: [[TMP377:%.*]] = fmul fast <4 x float> [[WIDE_LOAD46]], [[TMP376]]
+; CHECK-NEXT: [[TMP378:%.*]] = fmul fast <4 x float> [[WIDE_LOAD47]], [[TMP377]]
+; CHECK-NEXT: [[TMP379]] = fadd fast <4 x float> [[VEC_PHI43]], [[TMP378]]
+; CHECK-NEXT: [[INDEX_NEXT48]] = add nuw i64 [[IV1]], 4
+; CHECK-NEXT: [[TMP380:%.*]] = icmp eq i64 [[INDEX_NEXT48]], [[N_VEC39]]
+; CHECK-NEXT: br i1 [[TMP380]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP381:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP337]])
+; CHECK-NEXT: [[TMP382:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP358]])
+; CHECK-NEXT: [[TMP383:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP379]])
+; CHECK-NEXT: [[CMP_N49:%.*]] = icmp eq i64 [[SIZE]], [[N_VEC39]]
+; CHECK-NEXT: br i1 [[CMP_N49]], label %[[EXIT]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC39]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX50:%.*]] = phi float [ [[TMP381]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP300]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX51:%.*]] = phi float [ [[TMP382]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP301]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX52:%.*]] = phi float [ [[TMP383]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP302]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RDX_0:%.*]] = phi float [ [[BC_MERGE_RDX50]], %[[SCALAR_PH]] ], [ [[RDX_0_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RDX_1:%.*]] = phi float [ [[BC_MERGE_RDX51]], %[[SCALAR_PH]] ], [ [[RDX_1_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RED_2:%.*]] = phi float [ [[BC_MERGE_RDX52]], %[[SCALAR_PH]] ], [ [[RDX_2_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[ADD2:%.*]] = add i64 [[IV]], [[OFFSET]]
+; CHECK-NEXT: [[MUL2:%.*]] = mul i64 [[ADD2]], 3
+; CHECK-NEXT: [[GEP_SRC_DATA2:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[MUL2]]
+; CHECK-NEXT: [[TMP154:%.*]] = load float, ptr [[GEP_SRC_DATA2]], align 4
+; CHECK-NEXT: [[GEP_KERNEL5:%.*]] = getelementptr inbounds [512 x float], ptr @kernel, i64 0, i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[GEP_KERNEL5]], align 4
; CHECK-NEXT: [[MUL3:%.*]] = fmul fast float [[TMP154]], [[TMP1]]
-; CHECK-NEXT: [[GEP_KERNEL2:%.*]] = getelementptr inbounds [512 x float], ptr @kernel2, i64 0, i64 [[IV1]]
+; CHECK-NEXT: [[GEP_KERNEL2:%.*]] = getelementptr inbounds [512 x float], ptr @kernel2, i64 0, i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_KERNEL2]], align 4
; CHECK-NEXT: [[MUL5:%.*]] = fmul fast float [[MUL3]], [[TMP2]]
-; CHECK-NEXT: [[GEP_KERNEL3:%.*]] = getelementptr inbounds [512 x float], ptr @kernel3, i64 0, i64 [[IV1]]
+; CHECK-NEXT: [[GEP_KERNEL3:%.*]] = getelementptr inbounds [512 x float], ptr @kernel3, i64 0, i64 [[IV]]
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_KERNEL3]], align 4
; CHECK-NEXT: [[MUL7:%.*]] = fmul fast float [[MUL5]], [[TMP3]]
-; CHECK-NEXT: [[GEP_KERNEL4:%.*]] = getelementptr inbounds [512 x float], ptr @kernel4, i64 0, i64 [[IV1]]
+; CHECK-NEXT: [[GEP_KERNEL4:%.*]] = getelementptr inbounds [512 x float], ptr @kernel4, i64 0, i64 [[IV]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_KERNEL4]], align 4
; CHECK-NEXT: [[MUL9:%.*]] = fmul fast float [[MUL7]], [[TMP4]]
; CHECK-NEXT: [[RDX_0_NEXT]] = fadd fast float [[RDX_0]], [[MUL9]]
-; CHECK-NEXT: [[GEP_SRC_DATA_SUM:%.*]] = add i64 [[MUL1]], 1
+; CHECK-NEXT: [[GEP_SRC_DATA_SUM:%.*]] = add i64 [[MUL2]], 1
; CHECK-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[GEP_SRC_DATA_SUM]]
; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[ARRAYIDX11]], align 4
; CHECK-NEXT: [[MUL13:%.*]] = fmul fast float [[TMP1]], [[TMP5]]
@@ -233,7 +512,7 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[MUL17:%.*]] = fmul fast float [[TMP3]], [[MUL15]]
; CHECK-NEXT: [[MUL19:%.*]] = fmul fast float [[TMP4]], [[MUL17]]
; CHECK-NEXT: [[RDX_1_NEXT]] = fadd fast float [[RDX_1]], [[MUL19]]
-; CHECK-NEXT: [[GEP_SRC_DATA_SUM52:%.*]] = add i64 [[MUL1]], 2
+; CHECK-NEXT: [[GEP_SRC_DATA_SUM52:%.*]] = add i64 [[MUL2]], 2
; CHECK-NEXT: [[ARRAYIDX21:%.*]] = getelementptr inbounds [1536 x float], ptr @src_data, i64 0, i64 [[GEP_SRC_DATA_SUM52]]
; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[ARRAYIDX21]], align 4
; CHECK-NEXT: [[MUL23:%.*]] = fmul fast float [[TMP1]], [[TMP6]]
@@ -241,13 +520,13 @@ define float @_Z4testmm(i64 %size, i64 %offset) {
; CHECK-NEXT: [[MUL27:%.*]] = fmul fast float [[TMP3]], [[MUL25]]
; CHECK-NEXT: [[MUL29:%.*]] = fmul fast float [[TMP4]], [[MUL27]]
; CHECK-NEXT: [[RDX_2_NEXT]] = fadd fast float [[RED_2]], [[MUL29]]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[SIZE]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RDX_0_NEXT_LCSSA:%.*]] = phi float [ [[RDX_0_NEXT]], %[[LOOP]] ], [ [[TMP151]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[RDX_1_NEXT_LCSSA:%.*]] = phi float [ [[RDX_1_NEXT]], %[[LOOP]] ], [ [[TMP152]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[RDX_2_NEXT_LCSSA:%.*]] = phi float [ [[RDX_2_NEXT]], %[[LOOP]] ], [ [[TMP153]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RDX_0_NEXT_LCSSA:%.*]] = phi float [ [[RDX_0_NEXT]], %[[LOOP1]] ], [ [[TMP300]], %[[MIDDLE_BLOCK]] ], [ [[TMP381]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RDX_1_NEXT_LCSSA:%.*]] = phi float [ [[RDX_1_NEXT]], %[[LOOP1]] ], [ [[TMP301]], %[[MIDDLE_BLOCK]] ], [ [[TMP382]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RDX_2_NEXT_LCSSA:%.*]] = phi float [ [[RDX_2_NEXT]], %[[LOOP1]] ], [ [[TMP302]], %[[MIDDLE_BLOCK]] ], [ [[TMP383]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[RES_0:%.*]] = fadd float [[RDX_0_NEXT_LCSSA]], [[RDX_1_NEXT_LCSSA]]
; CHECK-NEXT: [[RES_1:%.*]] = fadd float [[RES_0]], [[RDX_2_NEXT_LCSSA]]
; CHECK-NEXT: ret float [[RES_1]]
@@ -306,5 +585,7 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[PROF3]] = !{!"branch_weights", i32 4, i32 12}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index f8af1f99d2361..b49e1aa5b4e46 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -9,12 +9,12 @@ define i32 @multi_exit_iv_uniform(i32 %a, i64 %N, ptr %dst) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 2147483648)
; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[UMIN]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 8, i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 16, i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -24,22 +24,32 @@ define i32 @multi_exit_iv_uniform(i32 %a, i64 %N, ptr %dst) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP4]], i64 8
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[TMP4]], i64 12
; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP5]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP9]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP12]], align 8
; CHECK-NEXT: [[TMP6]] = add <4 x i32> [[VEC_PHI]], splat (i32 -1)
; CHECK-NEXT: [[TMP7]] = add <4 x i32> [[VEC_PHI1]], splat (i32 -1)
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP10]] = add <4 x i32> [[VEC_PHI2]], splat (i32 -1)
+; CHECK-NEXT: [[TMP11]] = add <4 x i32> [[VEC_PHI3]], splat (i32 -1)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP7]], [[TMP6]]
-; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX4:%.*]] = add <4 x i32> [[TMP10]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX5:%.*]] = add <4 x i32> [[TMP11]], [[BIN_RDX4]]
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX5]])
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 555d41651156a..94dde2b58a0a0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -372,13 +372,16 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-LABEL: define i32 @load_factor_4_with_gap(
; CHECK-NOTF-SAME: i64 [[N:%.*]], ptr noalias [[A:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-NOTF: [[VECTOR_PH]]:
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -386,11 +389,23 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP42:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
; CHECK-NOTF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
; CHECK-NOTF-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NOTF-NEXT: [[TMP22:%.*]] = shl i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], 0
+; CHECK-NOTF-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], 1
+; CHECK-NOTF-NEXT: [[TMP31:%.*]] = add i64 [[INDEX]], [[TMP24]]
+; CHECK-NOTF-NEXT: [[TMP32:%.*]] = mul i64 [[TMP2]], 3
+; CHECK-NOTF-NEXT: [[TMP35:%.*]] = add i64 [[TMP32]], 0
+; CHECK-NOTF-NEXT: [[TMP36:%.*]] = mul i64 [[TMP35]], 1
+; CHECK-NOTF-NEXT: [[TMP39:%.*]] = add i64 [[INDEX]], [[TMP36]]
; CHECK-NOTF-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 0
; CHECK-NOTF-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP6]], i32 0
+; CHECK-NOTF-NEXT: [[TMP40:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP31]], i32 0
+; CHECK-NOTF-NEXT: [[TMP43:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP39]], i32 0
; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 4
; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
; CHECK-NOTF-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
@@ -401,41 +416,87 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 0
; CHECK-NOTF-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 1
; CHECK-NOTF-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 3
+; CHECK-NOTF-NEXT: [[WIDE_VEC7:%.*]] = load <vscale x 16 x i32>, ptr [[TMP40]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC8:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC7]])
+; CHECK-NOTF-NEXT: [[TMP25:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC8]], 0
+; CHECK-NOTF-NEXT: [[TMP26:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC8]], 1
+; CHECK-NOTF-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC8]], 3
+; CHECK-NOTF-NEXT: [[WIDE_VEC9:%.*]] = load <vscale x 16 x i32>, ptr [[TMP43]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC10:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC9]])
+; CHECK-NOTF-NEXT: [[TMP28:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 0
+; CHECK-NOTF-NEXT: [[TMP29:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 1
+; CHECK-NOTF-NEXT: [[TMP30:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 3
; CHECK-NOTF-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP9]]
; CHECK-NOTF-NEXT: [[TMP16:%.*]] = add <vscale x 4 x i32> [[VEC_PHI1]], [[TMP12]]
+; CHECK-NOTF-NEXT: [[TMP33:%.*]] = add <vscale x 4 x i32> [[VEC_PHI3]], [[TMP25]]
+; CHECK-NOTF-NEXT: [[TMP34:%.*]] = add <vscale x 4 x i32> [[VEC_PHI4]], [[TMP28]]
; CHECK-NOTF-NEXT: [[TMP17:%.*]] = add <vscale x 4 x i32> [[TMP15]], [[TMP10]]
; CHECK-NOTF-NEXT: [[TMP18:%.*]] = add <vscale x 4 x i32> [[TMP16]], [[TMP13]]
+; CHECK-NOTF-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[TMP33]], [[TMP26]]
+; CHECK-NOTF-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], [[TMP29]]
; CHECK-NOTF-NEXT: [[TMP19]] = add <vscale x 4 x i32> [[TMP17]], [[TMP11]]
; CHECK-NOTF-NEXT: [[TMP20]] = add <vscale x 4 x i32> [[TMP18]], [[TMP14]]
+; CHECK-NOTF-NEXT: [[TMP41]] = add <vscale x 4 x i32> [[TMP37]], [[TMP27]]
+; CHECK-NOTF-NEXT: [[TMP42]] = add <vscale x 4 x i32> [[TMP38]], [[TMP30]]
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NOTF-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP20]], [[TMP19]]
-; CHECK-NOTF-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NOTF-NEXT: [[BIN_RDX11:%.*]] = add <vscale x 4 x i32> [[TMP41]], [[BIN_RDX]]
+; CHECK-NOTF-NEXT: [[BIN_RDX12:%.*]] = add <vscale x 4 x i32> [[TMP42]], [[BIN_RDX11]]
+; CHECK-NOTF-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX12]])
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
+; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10:![0-9]+]]
; CHECK-NOTF: [[SCALAR_PH]]:
-; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP44]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NOTF-NEXT: [[N_MOD_VF13:%.*]] = urem i64 [[N]], 2
+; CHECK-NOTF-NEXT: [[N_VEC14:%.*]] = sub i64 [[N]], [[N_MOD_VF13]]
+; CHECK-NOTF-NEXT: [[TMP45:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
-; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT21:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <2 x i32> [ [[TMP45]], %[[SCALAR_PH]] ], [ [[TMP49:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
-; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <8 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 1, i32 5>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 3, i32 7>
+; CHECK-NOTF-NEXT: [[TMP47:%.*]] = add <2 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
+; CHECK-NOTF-NEXT: [[TMP48:%.*]] = add <2 x i32> [[TMP47]], [[STRIDED_VEC19]]
+; CHECK-NOTF-NEXT: [[TMP49]] = add <2 x i32> [[TMP48]], [[STRIDED_VEC20]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT21]] = add nuw i64 [[IV]], 2
+; CHECK-NOTF-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT21]], [[N_VEC14]]
+; CHECK-NOTF-NEXT: br i1 [[TMP50]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP49]])
+; CHECK-NOTF-NEXT: [[CMP_N22:%.*]] = icmp eq i64 [[N]], [[N_VEC14]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N22]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX23:%.*]] = phi i32 [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP44]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
+; CHECK-NOTF: [[LOOP1]]:
+; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX23]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NOTF-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 0
+; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[LD1]]
-; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 1
; CHECK-NOTF-NEXT: [[LD2:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
; CHECK-NOTF-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[LD2]]
-; CHECK-NOTF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3
+; CHECK-NOTF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 3
; CHECK-NOTF-NEXT: [[LD3:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; CHECK-NOTF-NEXT: [[RDX_NEXT]] = add nsw i32 [[ADD1]], [[LD3]]
-; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-NOTF: [[EXIT]]:
-; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP44]], %[[MIDDLE_BLOCK]] ], [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NOTF-NEXT: ret i32 [[RDX_NEXT_LCSSA]]
;
; CHECK-TF-LABEL: define i32 @load_factor_4_with_gap(
@@ -547,7 +608,7 @@ define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[TMP2]], ptr [[TMP14]], align 4
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NOTF-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -564,7 +625,7 @@ define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[IV]], ptr [[ARRAYIDX2]], align 4
; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-NOTF: [[EXIT]]:
; CHECK-NOTF-NEXT: ret void
;
@@ -620,13 +681,16 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-LABEL: define i32 @load_factor_4_with_tail_gap(
; CHECK-NOTF-SAME: i64 [[N:%.*]], ptr noalias [[A:%.*]]) #[[ATTR2]] {
; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], 2
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], [[TMP1]]
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-NOTF: [[VECTOR_PH]]:
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-NOTF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; CHECK-NOTF-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
@@ -636,11 +700,23 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP43:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP44:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[TMP6:%.*]] = add i64 [[TMP2]], 0
; CHECK-NOTF-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 1
; CHECK-NOTF-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], [[TMP7]]
+; CHECK-NOTF-NEXT: [[TMP24:%.*]] = shl i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[TMP25:%.*]] = add i64 [[TMP24]], 0
+; CHECK-NOTF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 1
+; CHECK-NOTF-NEXT: [[TMP33:%.*]] = add i64 [[INDEX]], [[TMP26]]
+; CHECK-NOTF-NEXT: [[TMP34:%.*]] = mul i64 [[TMP2]], 3
+; CHECK-NOTF-NEXT: [[TMP37:%.*]] = add i64 [[TMP34]], 0
+; CHECK-NOTF-NEXT: [[TMP38:%.*]] = mul i64 [[TMP37]], 1
+; CHECK-NOTF-NEXT: [[TMP41:%.*]] = add i64 [[INDEX]], [[TMP38]]
; CHECK-NOTF-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[INDEX]], i32 0
; CHECK-NOTF-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP8]], i32 0
+; CHECK-NOTF-NEXT: [[TMP42:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP33]], i32 0
+; CHECK-NOTF-NEXT: [[TMP45:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[TMP41]], i32 0
; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP9]], align 4
; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
; CHECK-NOTF-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
@@ -651,40 +727,87 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 0
; CHECK-NOTF-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 1
; CHECK-NOTF-NEXT: [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 2
+; CHECK-NOTF-NEXT: [[WIDE_VEC7:%.*]] = load <vscale x 16 x i32>, ptr [[TMP42]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC8:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC7]])
+; CHECK-NOTF-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC8]], 0
+; CHECK-NOTF-NEXT: [[TMP28:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC8]], 1
+; CHECK-NOTF-NEXT: [[TMP29:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC8]], 2
+; CHECK-NOTF-NEXT: [[WIDE_VEC9:%.*]] = load <vscale x 16 x i32>, ptr [[TMP45]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC10:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC9]])
+; CHECK-NOTF-NEXT: [[TMP30:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 0
+; CHECK-NOTF-NEXT: [[TMP31:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 1
+; CHECK-NOTF-NEXT: [[TMP32:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 2
; CHECK-NOTF-NEXT: [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP11]]
; CHECK-NOTF-NEXT: [[TMP18:%.*]] = add <vscale x 4 x i32> [[VEC_PHI1]], [[TMP14]]
+; CHECK-NOTF-NEXT: [[TMP35:%.*]] = add <vscale x 4 x i32> [[VEC_PHI3]], [[TMP27]]
+; CHECK-NOTF-NEXT: [[TMP36:%.*]] = add <vscale x 4 x i32> [[VEC_PHI4]], [[TMP30]]
; CHECK-NOTF-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[TMP12]]
; CHECK-NOTF-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[TMP18]], [[TMP15]]
+; CHECK-NOTF-NEXT: [[TMP39:%.*]] = add <vscale x 4 x i32> [[TMP35]], [[TMP28]]
+; CHECK-NOTF-NEXT: [[TMP40:%.*]] = add <vscale x 4 x i32> [[TMP36]], [[TMP31]]
; CHECK-NOTF-NEXT: [[TMP21]] = add <vscale x 4 x i32> [[TMP19]], [[TMP13]]
; CHECK-NOTF-NEXT: [[TMP22]] = add <vscale x 4 x i32> [[TMP20]], [[TMP16]]
+; CHECK-NOTF-NEXT: [[TMP43]] = add <vscale x 4 x i32> [[TMP39]], [[TMP29]]
+; CHECK-NOTF-NEXT: [[TMP44]] = add <vscale x 4 x i32> [[TMP40]], [[TMP32]]
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NOTF-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP22]], [[TMP21]]
-; CHECK-NOTF-NEXT: [[TMP24:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-NOTF-NEXT: br label %[[SCALAR_PH]]
+; CHECK-NOTF-NEXT: [[BIN_RDX11:%.*]] = add <vscale x 4 x i32> [[TMP43]], [[BIN_RDX]]
+; CHECK-NOTF-NEXT: [[BIN_RDX12:%.*]] = add <vscale x 4 x i32> [[TMP44]], [[BIN_RDX11]]
+; CHECK-NOTF-NEXT: [[TMP46:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX12]])
+; CHECK-NOTF-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5]], 2
+; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
-; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP24]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP46]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NOTF-NEXT: [[N_MOD_VF13:%.*]] = urem i64 [[N]], 2
+; CHECK-NOTF-NEXT: [[TMP47:%.*]] = icmp eq i64 [[N_MOD_VF13]], 0
+; CHECK-NOTF-NEXT: [[TMP48:%.*]] = select i1 [[TMP47]], i64 2, i64 [[N_MOD_VF13]]
+; CHECK-NOTF-NEXT: [[N_VEC14:%.*]] = sub i64 [[N]], [[TMP48]]
+; CHECK-NOTF-NEXT: [[TMP49:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
-; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT21:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <2 x i32> [ [[TMP49]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
-; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <8 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 1, i32 5>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 2, i32 6>
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <2 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
+; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <2 x i32> [[TMP51]], [[STRIDED_VEC19]]
+; CHECK-NOTF-NEXT: [[TMP53]] = add <2 x i32> [[TMP52]], [[STRIDED_VEC20]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT21]] = add nuw i64 [[IV]], 2
+; CHECK-NOTF-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT21]], [[N_VEC14]]
+; CHECK-NOTF-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP53]])
+; CHECK-NOTF-NEXT: br label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i32 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP46]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
+; CHECK-NOTF: [[LOOP1]]:
+; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX22]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NOTF-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 0
+; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[LD1]]
-; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 1
; CHECK-NOTF-NEXT: [[LD2:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
; CHECK-NOTF-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[LD2]]
-; CHECK-NOTF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2
+; CHECK-NOTF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 2
; CHECK-NOTF-NEXT: [[LD3:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; CHECK-NOTF-NEXT: [[RDX_NEXT]] = add nsw i32 [[ADD1]], [[LD3]]
-; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK-NOTF: [[EXIT]]:
-; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ]
; CHECK-NOTF-NEXT: ret i32 [[RDX_NEXT_LCSSA]]
;
; CHECK-TF-LABEL: define i32 @load_factor_4_with_tail_gap(
@@ -785,7 +908,7 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[TMP2]], ptr [[TMP14]], align 4
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NOTF-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -802,7 +925,7 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[IV]], ptr [[ARRAYIDX2]], align 4
; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK-NOTF: [[EXIT]]:
; CHECK-NOTF-NEXT: ret void
;
@@ -863,13 +986,16 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: br i1 [[CMP]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-NOTF: [[LOOP_PREHEADER]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-NOTF: [[VECTOR_PH]]:
; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NOTF-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NOTF-NEXT: [[TMP5:%.*]] = sub i64 [[N]], [[N_VEC]]
@@ -878,6 +1004,8 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP42:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP43:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NOTF-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[TMP6]]
; CHECK-NOTF-NEXT: [[TMP8:%.*]] = sub nuw nsw i64 [[TMP3]], 1
@@ -886,6 +1014,12 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[TMP11:%.*]] = mul i64 -4, [[TMP3]]
; CHECK-NOTF-NEXT: [[TMP12:%.*]] = add i64 [[TMP9]], [[TMP11]]
; CHECK-NOTF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP12]]
+; CHECK-NOTF-NEXT: [[TMP32:%.*]] = mul i64 -8, [[TMP3]]
+; CHECK-NOTF-NEXT: [[TMP33:%.*]] = add i64 [[TMP9]], [[TMP32]]
+; CHECK-NOTF-NEXT: [[TMP36:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP33]]
+; CHECK-NOTF-NEXT: [[TMP37:%.*]] = mul i64 -12, [[TMP3]]
+; CHECK-NOTF-NEXT: [[TMP40:%.*]] = add i64 [[TMP9]], [[TMP37]]
+; CHECK-NOTF-NEXT: [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP40]]
; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP10]], align 4
; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
; CHECK-NOTF-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
@@ -902,41 +1036,99 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[REVERSE7:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP18]])
; CHECK-NOTF-NEXT: [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC5]], 3
; CHECK-NOTF-NEXT: [[REVERSE8:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-NOTF-NEXT: [[WIDE_VEC12:%.*]] = load <vscale x 16 x i32>, ptr [[TMP36]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC13:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC12]])
+; CHECK-NOTF-NEXT: [[TMP44:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC13]], 0
+; CHECK-NOTF-NEXT: [[REVERSE14:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP44]])
+; CHECK-NOTF-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC13]], 1
+; CHECK-NOTF-NEXT: [[REVERSE15:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP27]])
+; CHECK-NOTF-NEXT: [[TMP28:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC13]], 3
+; CHECK-NOTF-NEXT: [[REVERSE16:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP28]])
+; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP41]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC17]])
+; CHECK-NOTF-NEXT: [[TMP29:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC18]], 0
+; CHECK-NOTF-NEXT: [[REVERSE19:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP29]])
+; CHECK-NOTF-NEXT: [[TMP30:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC18]], 1
+; CHECK-NOTF-NEXT: [[REVERSE20:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP30]])
+; CHECK-NOTF-NEXT: [[TMP31:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC18]], 3
+; CHECK-NOTF-NEXT: [[REVERSE21:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP31]])
; CHECK-NOTF-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[REVERSE]], [[VEC_PHI]]
; CHECK-NOTF-NEXT: [[TMP21:%.*]] = add <vscale x 4 x i32> [[REVERSE6]], [[VEC_PHI1]]
+; CHECK-NOTF-NEXT: [[TMP34:%.*]] = add <vscale x 4 x i32> [[REVERSE14]], [[VEC_PHI3]]
+; CHECK-NOTF-NEXT: [[TMP35:%.*]] = add <vscale x 4 x i32> [[REVERSE19]], [[VEC_PHI4]]
; CHECK-NOTF-NEXT: [[TMP22:%.*]] = add <vscale x 4 x i32> [[TMP20]], [[REVERSE2]]
; CHECK-NOTF-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP21]], [[REVERSE7]]
+; CHECK-NOTF-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], [[REVERSE15]]
+; CHECK-NOTF-NEXT: [[TMP39:%.*]] = add <vscale x 4 x i32> [[TMP35]], [[REVERSE20]]
; CHECK-NOTF-NEXT: [[TMP24]] = add <vscale x 4 x i32> [[TMP22]], [[REVERSE3]]
; CHECK-NOTF-NEXT: [[TMP25]] = add <vscale x 4 x i32> [[TMP23]], [[REVERSE8]]
+; CHECK-NOTF-NEXT: [[TMP42]] = add <vscale x 4 x i32> [[TMP38]], [[REVERSE16]]
+; CHECK-NOTF-NEXT: [[TMP43]] = add <vscale x 4 x i32> [[TMP39]], [[REVERSE21]]
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; CHECK-NOTF-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP25]], [[TMP24]]
-; CHECK-NOTF-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NOTF-NEXT: [[BIN_RDX22:%.*]] = add <vscale x 4 x i32> [[TMP42]], [[BIN_RDX]]
+; CHECK-NOTF-NEXT: [[BIN_RDX23:%.*]] = add <vscale x 4 x i32> [[TMP43]], [[BIN_RDX22]]
+; CHECK-NOTF-NEXT: [[TMP45:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX23]])
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
+; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
-; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NOTF-NEXT: [[N_MOD_VF24:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[N_VEC25:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF24]]
+; CHECK-NOTF-NEXT: [[TMP46:%.*]] = sub i64 [[N]], [[N_VEC25]]
+; CHECK-NOTF-NEXT: [[TMP47:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
-; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[INDEX26:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT35:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI27:%.*]] = phi <2 x i32> [ [[TMP47]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[IV:%.*]] = sub i64 [[N]], [[INDEX26]]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV]]
-; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 -4
+; CHECK-NOTF-NEXT: [[WIDE_VEC28:%.*]] = load <8 x i32>, ptr [[TMP50]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <8 x i32> [[WIDE_VEC28]], <8 x i32> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NOTF-NEXT: [[REVERSE30:%.*]] = shufflevector <2 x i32> [[STRIDED_VEC29]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC31:%.*]] = shufflevector <8 x i32> [[WIDE_VEC28]], <8 x i32> poison, <2 x i32> <i32 1, i32 5>
+; CHECK-NOTF-NEXT: [[REVERSE32:%.*]] = shufflevector <2 x i32> [[STRIDED_VEC31]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC33:%.*]] = shufflevector <8 x i32> [[WIDE_VEC28]], <8 x i32> poison, <2 x i32> <i32 3, i32 7>
+; CHECK-NOTF-NEXT: [[REVERSE34:%.*]] = shufflevector <2 x i32> [[STRIDED_VEC33]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <2 x i32> [[REVERSE30]], [[VEC_PHI27]]
+; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <2 x i32> [[TMP51]], [[REVERSE32]]
+; CHECK-NOTF-NEXT: [[TMP53]] = add <2 x i32> [[TMP52]], [[REVERSE34]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT35]] = add nuw i64 [[INDEX26]], 2
+; CHECK-NOTF-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT35]], [[N_VEC25]]
+; CHECK-NOTF-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP53]])
+; CHECK-NOTF-NEXT: [[CMP_N36:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC25]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N36]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL37:%.*]] = phi i64 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX38:%.*]] = phi i32 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
+; CHECK-NOTF: [[LOOP1]]:
+; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL37]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX38]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV1]]
+; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[LD1]], [[RDX]]
-; CHECK-NOTF-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 4
+; CHECK-NOTF-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX1]], i64 4
; CHECK-NOTF-NEXT: [[LD2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
; CHECK-NOTF-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD]], [[LD2]]
-; CHECK-NOTF-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 12
+; CHECK-NOTF-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX1]], i64 12
; CHECK-NOTF-NEXT: [[LD3:%.*]] = load i32, ptr [[ARRAYIDX8]], align 4
; CHECK-NOTF-NEXT: [[RDX_NEXT]] = add nsw i32 [[ADD2]], [[LD3]]
-; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
-; CHECK-NOTF-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NOTF-NEXT: br i1 [[CMP_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nsw i64 [[IV1]], -1
+; CHECK-NOTF-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[IV1]], 0
+; CHECK-NOTF-NEXT: br i1 [[CMP_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP1]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK-NOTF: [[EXIT_LOOPEXIT]]:
-; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP27]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP45]], %[[MIDDLE_BLOCK]] ], [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NOTF-NEXT: br label %[[EXIT]]
; CHECK-NOTF: [[EXIT]]:
; CHECK-NOTF-NEXT: [[RDX_FINAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -1057,7 +1249,7 @@ define void @store_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[TMP9]], ptr [[TMP21]], align 4
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NOTF-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
@@ -1075,7 +1267,7 @@ define void @store_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[IV_TRUNC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK-NOTF: [[EXIT_LOOPEXIT]]:
; CHECK-NOTF-NEXT: br label %[[EXIT]]
; CHECK-NOTF: [[EXIT]]:
@@ -1200,7 +1392,7 @@ define i32 @load_factor_4_with_tail_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[TMP41]] = add i32 [[TMP29]], [[TMP37]]
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NOTF-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP39]], [[TMP38]]
; CHECK-NOTF-NEXT: [[BIN_RDX4:%.*]] = add i32 [[TMP40]], [[BIN_RDX]]
@@ -1225,7 +1417,7 @@ define i32 @load_factor_4_with_tail_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[RDX_NEXT]] = add nsw i32 [[ADD1]], [[LD3]]
; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK-NOTF: [[EXIT_LOOPEXIT]]:
; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[BIN_RDX5]], %[[MIDDLE_BLOCK]] ]
; CHECK-NOTF-NEXT: br label %[[EXIT]]
@@ -1349,7 +1541,7 @@ define void @store_factor_4_with_tail_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[TMP9]], ptr [[TMP21]], align 4
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NOTF-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
@@ -1367,7 +1559,7 @@ define void @store_factor_4_with_tail_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: store i32 [[IV_TRUNC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK-NOTF: [[EXIT_LOOPEXIT]]:
; CHECK-NOTF-NEXT: br label %[[EXIT]]
; CHECK-NOTF: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
index 8a780eea300fc..318afbf9e5ffa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
@@ -91,46 +91,82 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
; INTERLEAVE-4-NEXT: ret i32 [[RED_NEXT_LCSSA]]
;
; INTERLEAVE-2-LABEL: @interleave_integer_reduction(
-; INTERLEAVE-2-NEXT: entry:
-; INTERLEAVE-2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8
+; INTERLEAVE-2-NEXT: iter.check:
+; INTERLEAVE-2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
; INTERLEAVE-2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; INTERLEAVE-2: vector.main.loop.iter.check:
+; INTERLEAVE-2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; INTERLEAVE-2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
; INTERLEAVE-2: vector.ph:
-; INTERLEAVE-2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; INTERLEAVE-2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
; INTERLEAVE-2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; INTERLEAVE-2-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE-2: vector.body:
-; INTERLEAVE-2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-2-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-2-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]
; INTERLEAVE-2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
+; INTERLEAVE-2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 8
+; INTERLEAVE-2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 12
; INTERLEAVE-2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 1
; INTERLEAVE-2-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 1
+; INTERLEAVE-2-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP5]], align 1
+; INTERLEAVE-2-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP8]], align 1
; INTERLEAVE-2-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
; INTERLEAVE-2-NEXT: [[TMP3]] = add <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD2]]
-; INTERLEAVE-2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; INTERLEAVE-2-NEXT: [[TMP6]] = add <4 x i32> [[VEC_PHI3]], [[WIDE_LOAD6]]
+; INTERLEAVE-2-NEXT: [[TMP7]] = add <4 x i32> [[VEC_PHI4]], [[WIDE_LOAD7]]
+; INTERLEAVE-2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; INTERLEAVE-2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-2-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; INTERLEAVE-2: middle.block:
; INTERLEAVE-2-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP3]], [[TMP2]]
-; INTERLEAVE-2-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; INTERLEAVE-2-NEXT: [[BIN_RDX8:%.*]] = add <4 x i32> [[TMP6]], [[BIN_RDX]]
+; INTERLEAVE-2-NEXT: [[BIN_RDX9:%.*]] = add <4 x i32> [[TMP7]], [[BIN_RDX8]]
+; INTERLEAVE-2-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX9]])
; INTERLEAVE-2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; INTERLEAVE-2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; INTERLEAVE-2: scalar.ph:
-; INTERLEAVE-2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; INTERLEAVE-2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; INTERLEAVE-2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; INTERLEAVE-2: vec.epilog.iter.check:
+; INTERLEAVE-2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; INTERLEAVE-2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; INTERLEAVE-2: vec.epilog.ph:
+; INTERLEAVE-2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; INTERLEAVE-2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; INTERLEAVE-2-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[N]], 4
+; INTERLEAVE-2-NEXT: [[N_VEC11:%.*]] = sub i64 [[N]], [[N_MOD_VF10]]
+; INTERLEAVE-2-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; INTERLEAVE-2-NEXT: br label [[LOOP:%.*]]
+; INTERLEAVE-2: vec.epilog.vector.body:
+; INTERLEAVE-2-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[LOOP]] ]
+; INTERLEAVE-2-NEXT: [[VEC_PHI13:%.*]] = phi <4 x i32> [ [[TMP10]], [[VEC_EPILOG_PH]] ], [ [[TMP12:%.*]], [[LOOP]] ]
+; INTERLEAVE-2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX12]]
+; INTERLEAVE-2-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP11]], align 1
+; INTERLEAVE-2-NEXT: [[TMP12]] = add <4 x i32> [[VEC_PHI13]], [[WIDE_LOAD14]]
+; INTERLEAVE-2-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX12]], 4
+; INTERLEAVE-2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC11]]
+; INTERLEAVE-2-NEXT: br i1 [[TMP13]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; INTERLEAVE-2: vec.epilog.middle.block:
+; INTERLEAVE-2-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP12]])
+; INTERLEAVE-2-NEXT: [[CMP_N16:%.*]] = icmp eq i64 [[N]], [[N_VEC11]]
+; INTERLEAVE-2-NEXT: br i1 [[CMP_N16]], label [[EXIT]], label [[SCALAR_PH]]
+; INTERLEAVE-2: vec.epilog.scalar.ph:
+; INTERLEAVE-2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; INTERLEAVE-2-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP9]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
+; INTERLEAVE-2-NEXT: br label [[LOOP1:%.*]]
; INTERLEAVE-2: loop:
-; INTERLEAVE-2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; INTERLEAVE-2-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
+; INTERLEAVE-2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP1]] ]
+; INTERLEAVE-2-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX17]], [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP1]] ]
; INTERLEAVE-2-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
; INTERLEAVE-2-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 1
; INTERLEAVE-2-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[L]]
; INTERLEAVE-2-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; INTERLEAVE-2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; INTERLEAVE-2-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; INTERLEAVE-2-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; INTERLEAVE-2: exit:
-; INTERLEAVE-2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], [[LOOP]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; INTERLEAVE-2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], [[LOOP1]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ [[TMP14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
; INTERLEAVE-2-NEXT: ret i32 [[RED_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
index c4d06254a0d30..e62a10fb59c98 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
@@ -53,12 +53,16 @@ define i32 @add_reduction_select_operand_constant_but_non_uniform() {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 42, i32 0, i32 0, i32 0>, %[[VECTOR_PH]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[VEC_PHI2]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[VEC_PHI3]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 64
; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX4:%.*]] = add <4 x i32> [[VEC_PHI2]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX5:%.*]] = add <4 x i32> [[VEC_PHI3]], [[BIN_RDX4]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX5]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
index 53e649974afa7..4a663b75d6e52 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
@@ -141,12 +141,12 @@ define double @fdot_f32_f64(ptr %a, ptr %b) #0 {
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP6]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP6]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -154,28 +154,50 @@ define double @fdot_f32_f64(ptr %a, ptr %b) #0 {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x double> [ insertelement (<vscale x 4 x double> splat (double -0.000000e+00), double 0.000000e+00, i32 0), %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 4 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-NEXT: [[TMP19:%.*]] = mul nuw nsw i64 [[TMP6]], 3
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[TMP19]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 1
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP15]], align 1
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 4 x float>, ptr [[TMP22]], align 1
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x float>, ptr [[TMP23]], align 1
; CHECK-NEXT: [[TMP10:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD]] to <vscale x 4 x double>
; CHECK-NEXT: [[TMP13:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD2]] to <vscale x 4 x double>
+; CHECK-NEXT: [[TMP26:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD5]] to <vscale x 4 x double>
+; CHECK-NEXT: [[TMP27:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD6]] to <vscale x 4 x double>
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[TMP7]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr float, ptr [[TMP7]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr float, ptr [[TMP7]], i64 [[TMP19]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 1
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 1
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <vscale x 4 x float>, ptr [[TMP30]], align 1
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x float>, ptr [[TMP32]], align 1
; CHECK-NEXT: [[TMP9:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD3]] to <vscale x 4 x double>
; CHECK-NEXT: [[TMP12:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD4]] to <vscale x 4 x double>
+; CHECK-NEXT: [[TMP33:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD9]] to <vscale x 4 x double>
+; CHECK-NEXT: [[TMP21:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD10]] to <vscale x 4 x double>
; CHECK-NEXT: [[TMP11:%.*]] = fmul <vscale x 4 x double> [[TMP9]], [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = fmul <vscale x 4 x double> [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP24:%.*]] = fmul <vscale x 4 x double> [[TMP33]], [[TMP26]]
+; CHECK-NEXT: [[TMP25:%.*]] = fmul <vscale x 4 x double> [[TMP21]], [[TMP27]]
; CHECK-NEXT: [[TMP17]] = fadd reassoc contract <vscale x 4 x double> [[TMP11]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP16]] = fadd reassoc contract <vscale x 4 x double> [[TMP14]], [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP28]] = fadd reassoc contract <vscale x 4 x double> [[TMP24]], [[VEC_PHI2]]
+; CHECK-NEXT: [[TMP29]] = fadd reassoc contract <vscale x 4 x double> [[TMP25]], [[VEC_PHI3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc contract <vscale x 4 x double> [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP18:%.*]] = call reassoc contract double @llvm.vector.reduce.fadd.nxv4f64(double -0.000000e+00, <vscale x 4 x double> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX11:%.*]] = fadd reassoc contract <vscale x 4 x double> [[TMP28]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX12:%.*]] = fadd reassoc contract <vscale x 4 x double> [[TMP29]], [[BIN_RDX11]]
+; CHECK-NEXT: [[TMP31:%.*]] = call reassoc contract double @llvm.vector.reduce.fadd.nxv4f64(double -0.000000e+00, <vscale x 4 x double> [[BIN_RDX12]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
index c2579ad0a933b..087fe17540576 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
@@ -71,10 +71,10 @@ define i40 @partial_reduce_not_known_factor(i32 %a, i32 %b, i16 %N) {
; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 0)
; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i16 [[SMAX]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[B]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
@@ -89,14 +89,20 @@ define i40 @partial_reduce_not_known_factor(i32 %a, i32 %b, i16 %N) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i40> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i40> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i40> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i40> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6]] = or <2 x i40> [[VEC_PHI]], [[TMP5]]
; CHECK-NEXT: [[TMP8]] = or <2 x i40> [[VEC_PHI3]], [[TMP5]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP10]] = or <2 x i40> [[VEC_PHI4]], [[TMP5]]
+; CHECK-NEXT: [[TMP9]] = or <2 x i40> [[VEC_PHI5]], [[TMP5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <2 x i40> [[TMP8]], [[TMP6]]
-; CHECK-NEXT: [[TMP9:%.*]] = call i40 @llvm.vector.reduce.or.v2i40(<2 x i40> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX6:%.*]] = or <2 x i40> [[TMP10]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX7:%.*]] = or <2 x i40> [[TMP9]], [[BIN_RDX6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i40 @llvm.vector.reduce.or.v2i40(<2 x i40> [[BIN_RDX7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
index 4bb6bf39413d0..0065ecdec0112 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
@@ -4,39 +4,83 @@
define i64 @reduction(i64 %arg) #0 {
; CHECK-LABEL: define i64 @reduction(
; CHECK-SAME: i64 [[ARG:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ARG]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-NEXT: [[TMP5]] = or <4 x i32> [[VEC_PHI]], splat (i32 1)
; CHECK-NEXT: [[TMP1]] = or <4 x i32> [[VEC_PHI2]], splat (i32 1)
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-NEXT: [[TMP16]] = or <4 x i32> [[VEC_PHI4]], splat (i32 1)
+; CHECK-NEXT: [[TMP17]] = or <4 x i32> [[VEC_PHI3]], splat (i32 1)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 4)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP18:%.*]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 1)
+; CHECK-NEXT: [[TMP19:%.*]] = mul nsw <4 x i64> [[TMP18]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[BIN_RDX1:%.*]] = or <4 x i32> [[TMP1]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX1]])
+; CHECK-NEXT: [[BIN_RDX4:%.*]] = or <4 x i32> [[TMP16]], [[BIN_RDX1]]
+; CHECK-NEXT: [[BIN_RDX5:%.*]] = or <4 x i32> [[TMP17]], [[BIN_RDX4]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX5]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP19]], i64 3
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i64> poison, i64 [[ARG]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT8]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT9]], <i64 0, i64 1, i64 2, i64 3>
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT13:%.*]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VEC_IND11:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT14:%.*]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VEC_PHI12:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP10:%.*]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[TMP10]] = or <4 x i32> [[VEC_PHI12]], splat (i32 1)
+; CHECK-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT14]] = add nsw <4 x i64> [[VEC_IND11]], splat (i64 4)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT13]], 100
+; CHECK-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[SCALAR_PH]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i64> [[VEC_IND11]], splat (i64 1)
+; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <4 x i64> [[TMP12]], [[BROADCAST_SPLAT7]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP13]], i64 3
+; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX15:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 96, %[[SCALAR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[TMP7]], %[[SCALAR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[BC_MERGE_RDX15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[TMP2]] = or i32 [[VEC_PHI1]], 1
; CHECK-NEXT: [[TMP3]] = add nsw i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw i64 [[TMP3]], [[ARG]]
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[TMP3]], 100
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i64 [ [[TMP4]], %[[LOOP]] ]
-; CHECK-NEXT: [[BIN_RDX:%.*]] = phi i32 [ [[TMP2]], %[[LOOP]] ]
+; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i64 [ [[TMP4]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[BIN_RDX:%.*]] = phi i32 [ [[TMP2]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i32 [[BIN_RDX]] to i64
; CHECK-NEXT: [[RES:%.*]] = add i64 [[EXT]], [[MUL_LCSSA]]
; CHECK-NEXT: ret i64 [[RES]]
@@ -94,11 +138,11 @@ define i32 @or_reduction_with_freeze(ptr %dst, ptr %src) {
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP12:%.*]] = shl i64 [[N_VEC]], 3
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr [[DST]], align 8, !alias.scope [[META4:![0-9]+]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr [[DST]], align 8, !alias.scope [[META6:![0-9]+]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP24]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -106,29 +150,45 @@ define i32 @or_reduction_with_freeze(ptr %dst, ptr %src) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI9:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI10:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i64 2
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[NEXT_GEP]], align 8, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
-; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP15]], align 8, !alias.scope [[META7]], !noalias [[META4]]
-; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[NEXT_GEP]], align 8, !alias.scope [[META7]], !noalias [[META4]]
-; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP15]], align 8, !alias.scope [[META7]], !noalias [[META4]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i64 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i64 6
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[NEXT_GEP]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP15]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <2 x i64>, ptr [[TMP26]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[NEXT_GEP]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP15]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP23]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP26]], align 8, !alias.scope [[META9]], !noalias [[META6]]
; CHECK-NEXT: [[TMP16:%.*]] = or <2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP17:%.*]] = or <2 x i64> [[WIDE_LOAD9]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP27:%.*]] = or <2 x i64> [[WIDE_LOAD12]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP30:%.*]] = or <2 x i64> [[WIDE_LOAD13]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP18:%.*]] = freeze <2 x i64> [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = freeze <2 x i64> [[TMP17]]
+; CHECK-NEXT: [[TMP32:%.*]] = freeze <2 x i64> [[TMP27]]
+; CHECK-NEXT: [[TMP25:%.*]] = freeze <2 x i64> [[TMP30]]
; CHECK-NEXT: [[TMP20]] = or <2 x i64> [[TMP18]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP21]] = or <2 x i64> [[TMP19]], [[VEC_PHI8]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP28]] = or <2 x i64> [[TMP32]], [[VEC_PHI9]]
+; CHECK-NEXT: [[TMP29]] = or <2 x i64> [[TMP25]], [[VEC_PHI10]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <2 x i64> [[TMP21]], [[TMP20]]
-; CHECK-NEXT: [[TMP23:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX14:%.*]] = or <2 x i64> [[TMP28]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX15:%.*]] = or <2 x i64> [[TMP29]], [[BIN_RDX14]]
+; CHECK-NEXT: [[TMP31:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[BIN_RDX15]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP23]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[ENTRY]] ], [ [[SRC]], %[[VECTOR_SCEVCHECK]] ], [ [[SRC]], %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -142,9 +202,9 @@ define i32 @or_reduction_with_freeze(ptr %dst, ptr %src) {
; CHECK-NEXT: [[OR]] = or i64 [[FR]], [[MASK]]
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr i8, ptr [[PTR_IV]], i64 8
; CHECK-NEXT: [[CMP:%.*]] = icmp eq ptr [[PTR_IV]], [[DST]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[OR_LCSSA:%.*]] = phi i64 [ [[OR]], %[[LOOP]] ], [ [[TMP23]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[OR_LCSSA:%.*]] = phi i64 [ [[OR]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[CAST:%.*]] = trunc i64 [[OR_LCSSA]] to i32
; CHECK-NEXT: ret i32 [[CAST]]
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 882a55765e25e..ac69ea10bd1b6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -49,15 +49,13 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-LABEL: define i32 @chained_recurrences(
; VSCALEFORTUNING2-SAME: i32 [[X:%.*]], i64 [[Y:%.*]], ptr [[SRC_1:%.*]], i32 [[Z:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
; VSCALEFORTUNING2-NEXT: [[ENTRY:.*]]:
-; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; VSCALEFORTUNING2-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 16)
; VSCALEFORTUNING2-NEXT: [[TMP0:%.*]] = add i64 [[Y]], 1
+; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT: [[UMAX:%.*]] = shl nuw i64 [[TMP1]], 4
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[UMAX]]
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VSCALEFORTUNING2: [[VECTOR_PH]]:
-; VSCALEFORTUNING2-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 3
+; VSCALEFORTUNING2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
; VSCALEFORTUNING2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
; VSCALEFORTUNING2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
@@ -89,6 +87,8 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-NEXT: [[VECTOR_RECUR4:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT3]], %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP47:%.*]], %[[VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP48:%.*]], %[[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI6:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP65:%.*]], %[[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI7:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP66:%.*]], %[[VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP8]], align 4
; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP24]], i64 0
; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT7]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -96,36 +96,55 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-NEXT: [[TMP26]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT7]], <vscale x 4 x i32> [[BROADCAST_SPLAT7]], i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP27:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR4]], <vscale x 4 x i32> [[TMP25]], i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP28:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[TMP25]], <vscale x 4 x i32> [[TMP26]], i32 1)
+; VSCALEFORTUNING2-NEXT: [[TMP50:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[TMP26]], <vscale x 4 x i32> [[TMP26]], i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP29:%.*]] = or <vscale x 4 x i32> [[TMP27]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[TMP30:%.*]] = or <vscale x 4 x i32> [[TMP28]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT: [[TMP51:%.*]] = or <vscale x 4 x i32> [[TMP50]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[TMP31:%.*]] = shl <vscale x 4 x i32> [[TMP29]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP32:%.*]] = shl <vscale x 4 x i32> [[TMP30]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT: [[TMP67:%.*]] = shl <vscale x 4 x i32> [[TMP51]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP33:%.*]] = or <vscale x 4 x i32> [[TMP31]], splat (i32 2)
; VSCALEFORTUNING2-NEXT: [[TMP34:%.*]] = or <vscale x 4 x i32> [[TMP32]], splat (i32 2)
+; VSCALEFORTUNING2-NEXT: [[TMP68:%.*]] = or <vscale x 4 x i32> [[TMP67]], splat (i32 2)
; VSCALEFORTUNING2-NEXT: [[TMP35:%.*]] = or <vscale x 4 x i32> [[TMP11]], [[TMP33]]
; VSCALEFORTUNING2-NEXT: [[TMP36:%.*]] = or <vscale x 4 x i32> [[TMP11]], [[TMP34]]
+; VSCALEFORTUNING2-NEXT: [[TMP69:%.*]] = or <vscale x 4 x i32> [[TMP11]], [[TMP68]]
; VSCALEFORTUNING2-NEXT: [[TMP37:%.*]] = or <vscale x 4 x i32> [[TMP35]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[TMP38:%.*]] = or <vscale x 4 x i32> [[TMP36]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT: [[TMP70:%.*]] = or <vscale x 4 x i32> [[TMP69]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[DOTSPLAT]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER8:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[DOTSPLAT]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER11:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[DOTSPLAT]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[DOTSPLAT]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; VSCALEFORTUNING2-NEXT: [[TMP39:%.*]] = lshr <vscale x 4 x i32> [[TMP37]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP40:%.*]] = lshr <vscale x 4 x i32> [[TMP38]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT: [[TMP71:%.*]] = lshr <vscale x 4 x i32> [[TMP70]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP41:%.*]] = zext <vscale x 4 x i32> [[TMP39]] to <vscale x 4 x i64>
; VSCALEFORTUNING2-NEXT: [[TMP42:%.*]] = zext <vscale x 4 x i32> [[TMP40]] to <vscale x 4 x i64>
+; VSCALEFORTUNING2-NEXT: [[TMP72:%.*]] = zext <vscale x 4 x i32> [[TMP71]] to <vscale x 4 x i64>
; VSCALEFORTUNING2-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP41]]
; VSCALEFORTUNING2-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP42]]
+; VSCALEFORTUNING2-NEXT: [[WIDE_GEP15:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP72]]
; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP43]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP44]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER18:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP15]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER19:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP15]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; VSCALEFORTUNING2-NEXT: [[TMP45:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
; VSCALEFORTUNING2-NEXT: [[TMP46:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER8]], [[VEC_PHI5]]
+; VSCALEFORTUNING2-NEXT: [[TMP73:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER11]], [[VEC_PHI6]]
+; VSCALEFORTUNING2-NEXT: [[TMP74:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER12]], [[VEC_PHI7]]
; VSCALEFORTUNING2-NEXT: [[TMP47]] = or <vscale x 4 x i32> [[TMP45]], [[WIDE_MASKED_GATHER9]]
; VSCALEFORTUNING2-NEXT: [[TMP48]] = or <vscale x 4 x i32> [[TMP46]], [[WIDE_MASKED_GATHER10]]
+; VSCALEFORTUNING2-NEXT: [[TMP65]] = or <vscale x 4 x i32> [[TMP73]], [[WIDE_MASKED_GATHER18]]
+; VSCALEFORTUNING2-NEXT: [[TMP66]] = or <vscale x 4 x i32> [[TMP74]], [[WIDE_MASKED_GATHER19]]
; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; VSCALEFORTUNING2-NEXT: [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[TMP49]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VSCALEFORTUNING2: [[MIDDLE_BLOCK]]:
; VSCALEFORTUNING2-NEXT: [[BIN_RDX:%.*]] = or <vscale x 4 x i32> [[TMP48]], [[TMP47]]
-; VSCALEFORTUNING2-NEXT: [[TMP50:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; VSCALEFORTUNING2-NEXT: [[BIN_RDX20:%.*]] = or <vscale x 4 x i32> [[TMP65]], [[BIN_RDX]]
+; VSCALEFORTUNING2-NEXT: [[BIN_RDX21:%.*]] = or <vscale x 4 x i32> [[TMP66]], [[BIN_RDX20]]
+; VSCALEFORTUNING2-NEXT: [[TMP75:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[BIN_RDX21]])
; VSCALEFORTUNING2-NEXT: [[TMP52:%.*]] = call i32 @llvm.vscale.i32()
; VSCALEFORTUNING2-NEXT: [[TMP53:%.*]] = mul nuw i32 [[TMP52]], 4
; VSCALEFORTUNING2-NEXT: [[TMP54:%.*]] = sub i32 [[TMP53]], 1
@@ -134,13 +153,13 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VSCALEFORTUNING2: [[SCALAR_PH]]:
; VSCALEFORTUNING2-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP24]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; VSCALEFORTUNING2-NEXT: [[SCALAR_RECUR_INIT11:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VSCALEFORTUNING2-NEXT: [[SCALAR_RECUR_INIT22:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VSCALEFORTUNING2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP50]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP75]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VSCALEFORTUNING2-NEXT: br label %[[LOOP:.*]]
; VSCALEFORTUNING2: [[LOOP]]:
-; VSCALEFORTUNING2-NEXT: [[TMP51:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP57:%.*]], %[[LOOP]] ]
-; VSCALEFORTUNING2-NEXT: [[TMP55:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT11]], %[[SCALAR_PH]] ], [ [[TMP51]], %[[LOOP]] ]
+; VSCALEFORTUNING2-NEXT: [[TMP76:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP57:%.*]], %[[LOOP]] ]
+; VSCALEFORTUNING2-NEXT: [[TMP55:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT22]], %[[SCALAR_PH]] ], [ [[TMP76]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_2:%.*]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[TMP56:%.*]] = add i64 [[Y]], 1
@@ -170,7 +189,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[Y]]
; VSCALEFORTUNING2-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; VSCALEFORTUNING2: [[EXIT]]:
-; VSCALEFORTUNING2-NEXT: [[RED_2_LCSSA:%.*]] = phi i32 [ [[RED_2]], %[[LOOP]] ], [ [[TMP50]], %[[MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT: [[RED_2_LCSSA:%.*]] = phi i32 [ [[RED_2]], %[[LOOP]] ], [ [[TMP75]], %[[MIDDLE_BLOCK]] ]
; VSCALEFORTUNING2-NEXT: ret i32 [[RED_2_LCSSA]]
;
; PRED-LABEL: define i32 @chained_recurrences(
@@ -285,53 +304,101 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[ENTRY:.*]]:
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 1
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
+; DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP1]], 4
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP7]]
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; DEFAULT: [[VECTOR_PH1]]:
; DEFAULT-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 2
-; DEFAULT-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP5]], 1
+; DEFAULT-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP5]], 2
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
-; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[VEC_PHI4:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP13:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
+; DEFAULT-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP5]], 1
+; DEFAULT-NEXT: [[TMP10:%.*]] = mul nuw nsw i64 [[TMP5]], 3
; DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr i16, ptr [[TMP13]], i64 [[TMP5]]
+; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr i16, ptr [[TMP13]], i64 [[TMP9]]
+; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr i16, ptr [[TMP13]], i64 [[TMP10]]
; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP13]], align 2
; DEFAULT-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
+; DEFAULT-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x i16>, ptr [[TMP12]], align 2
+; DEFAULT-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 4 x i16>, ptr [[TMP14]], align 2
; DEFAULT-NEXT: [[TMP19:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; DEFAULT-NEXT: [[TMP20:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD2]], [[BROADCAST_SPLAT]]
+; DEFAULT-NEXT: [[TMP16:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD6]], [[BROADCAST_SPLAT]]
+; DEFAULT-NEXT: [[TMP17:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD7]], [[BROADCAST_SPLAT]]
; DEFAULT-NEXT: [[TMP21]] = or <vscale x 4 x i16> [[TMP19]], [[VEC_PHI]]
; DEFAULT-NEXT: [[TMP22]] = or <vscale x 4 x i16> [[TMP20]], [[VEC_PHI1]]
+; DEFAULT-NEXT: [[TMP27]] = or <vscale x 4 x i16> [[TMP16]], [[VEC_PHI3]]
+; DEFAULT-NEXT: [[TMP32]] = or <vscale x 4 x i16> [[TMP17]], [[VEC_PHI4]]
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; DEFAULT-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: [[BIN_RDX:%.*]] = or <vscale x 4 x i16> [[TMP22]], [[TMP21]]
-; DEFAULT-NEXT: [[TMP24:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[BIN_RDX]])
+; DEFAULT-NEXT: [[BIN_RDX8:%.*]] = or <vscale x 4 x i16> [[TMP27]], [[BIN_RDX]]
+; DEFAULT-NEXT: [[BIN_RDX9:%.*]] = or <vscale x 4 x i16> [[TMP32]], [[BIN_RDX8]]
+; DEFAULT-NEXT: [[TMP33:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[BIN_RDX9]])
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; DEFAULT: [[SCALAR_PH]]:
-; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; DEFAULT-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP24]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP6]]
+; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; DEFAULT: [[VEC_EPILOG_PH]]:
+; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; DEFAULT-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP33]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; DEFAULT-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
+; DEFAULT-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], [[TMP25]]
+; DEFAULT-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF10]]
+; DEFAULT-NEXT: [[TMP26:%.*]] = insertelement <vscale x 2 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[X]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT12]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
-; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[VEC_PHI15:%.*]] = phi <vscale x 2 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP29:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; DEFAULT-NEXT: [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; DEFAULT-NEXT: [[WIDE_LOAD16:%.*]] = load <vscale x 2 x i16>, ptr [[GEP]], align 2
+; DEFAULT-NEXT: [[TMP28:%.*]] = udiv <vscale x 2 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
+; DEFAULT-NEXT: [[TMP29]] = or <vscale x 2 x i16> [[TMP28]], [[VEC_PHI15]]
+; DEFAULT-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[IV]], [[TMP25]]
+; DEFAULT-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC11]]
+; DEFAULT-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: [[TMP31:%.*]] = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> [[TMP29]])
+; DEFAULT-NEXT: [[CMP_N18:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC11]]
+; DEFAULT-NEXT: br i1 [[CMP_N18]], label %[[EXIT]], label %[[SCALAR_PH]]
+; DEFAULT: [[SCALAR_PH]]:
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP33]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT: br label %[[LOOP1:.*]]
+; DEFAULT: [[LOOP1]]:
+; DEFAULT-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; DEFAULT-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX19]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP1]] ]
+; DEFAULT-NEXT: [[GEP1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV1]]
+; DEFAULT-NEXT: [[L:%.*]] = load i16, ptr [[GEP1]], align 2
; DEFAULT-NEXT: [[DIV:%.*]] = udiv i16 [[L]], [[X]]
; DEFAULT-NEXT: [[RED_NEXT]] = or i16 [[DIV]], [[RED]]
-; DEFAULT-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; DEFAULT-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; DEFAULT-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; DEFAULT-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
+; DEFAULT-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
+; DEFAULT-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; DEFAULT: [[EXIT]]:
-; DEFAULT-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP24]], %[[MIDDLE_BLOCK]] ]
+; DEFAULT-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP1]] ], [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; DEFAULT-NEXT: ret i16 [[RED_NEXT_LCSSA]]
;
; VSCALEFORTUNING2-LABEL: define i16 @reduce_udiv(
@@ -345,12 +412,12 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 1
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; VSCALEFORTUNING2: [[VECTOR_PH]]:
; VSCALEFORTUNING2-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP1]], 2
-; VSCALEFORTUNING2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP7]], 1
+; VSCALEFORTUNING2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP7]], 2
; VSCALEFORTUNING2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
; VSCALEFORTUNING2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
@@ -360,20 +427,34 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI4:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
+; VSCALEFORTUNING2-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP7]], 1
+; VSCALEFORTUNING2-NEXT: [[TMP10:%.*]] = mul nuw nsw i64 [[TMP7]], 3
; VSCALEFORTUNING2-NEXT: [[TMP12:%.*]] = getelementptr i16, ptr [[TMP8]], i64 [[TMP7]]
+; VSCALEFORTUNING2-NEXT: [[TMP18:%.*]] = getelementptr i16, ptr [[TMP8]], i64 [[TMP11]]
+; VSCALEFORTUNING2-NEXT: [[TMP28:%.*]] = getelementptr i16, ptr [[TMP8]], i64 [[TMP10]]
; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP8]], align 2
; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i16>, ptr [[TMP12]], align 2
+; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
+; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 4 x i16>, ptr [[TMP28]], align 2
; VSCALEFORTUNING2-NEXT: [[TMP13:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[TMP14:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD2]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT: [[TMP29:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD6]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT: [[TMP30:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD7]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[TMP15]] = or <vscale x 4 x i16> [[TMP13]], [[VEC_PHI]]
; VSCALEFORTUNING2-NEXT: [[TMP16]] = or <vscale x 4 x i16> [[TMP14]], [[VEC_PHI1]]
+; VSCALEFORTUNING2-NEXT: [[TMP22]] = or <vscale x 4 x i16> [[TMP29]], [[VEC_PHI3]]
+; VSCALEFORTUNING2-NEXT: [[TMP27]] = or <vscale x 4 x i16> [[TMP30]], [[VEC_PHI4]]
; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; VSCALEFORTUNING2-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; VSCALEFORTUNING2: [[MIDDLE_BLOCK]]:
; VSCALEFORTUNING2-NEXT: [[BIN_RDX:%.*]] = or <vscale x 4 x i16> [[TMP16]], [[TMP15]]
-; VSCALEFORTUNING2-NEXT: [[TMP18:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[BIN_RDX]])
+; VSCALEFORTUNING2-NEXT: [[BIN_RDX8:%.*]] = or <vscale x 4 x i16> [[TMP22]], [[BIN_RDX]]
+; VSCALEFORTUNING2-NEXT: [[BIN_RDX9:%.*]] = or <vscale x 4 x i16> [[TMP27]], [[BIN_RDX8]]
+; VSCALEFORTUNING2-NEXT: [[TMP31:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[BIN_RDX9]])
; VSCALEFORTUNING2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_ITER_CHECK]]:
@@ -381,7 +462,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_PH]]:
; VSCALEFORTUNING2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
; VSCALEFORTUNING2-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 1
; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], [[TMP20]]
@@ -406,11 +487,11 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; VSCALEFORTUNING2: [[VEC_EPILOG_SCALAR_PH]]:
; VSCALEFORTUNING2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX13:%.*]] = phi i16 [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: br label %[[LOOP:.*]]
; VSCALEFORTUNING2: [[LOOP]]:
; VSCALEFORTUNING2-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; VSCALEFORTUNING2-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX13]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; VSCALEFORTUNING2-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX19]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[GEP1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV1]]
; VSCALEFORTUNING2-NEXT: [[L:%.*]] = load i16, ptr [[GEP1]], align 2
; VSCALEFORTUNING2-NEXT: [[DIV:%.*]] = udiv i16 [[L]], [[X]]
@@ -419,7 +500,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
; VSCALEFORTUNING2-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; VSCALEFORTUNING2: [[EXIT]]:
-; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; VSCALEFORTUNING2-NEXT: ret i16 [[RED_NEXT_LCSSA]]
;
; PRED-LABEL: define i16 @reduce_udiv(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
index 5f43b71e1444c..75e30632c5cc8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
@@ -313,17 +313,23 @@ for.end:
; ADD (with reduction stored in invariant address)
-; CHECK-REMARK: vectorized loop (vectorization width: vscale x 4, interleaved count: 2)
+; CHECK-REMARK: vectorized loop (vectorization width: vscale x 4, interleaved count: 4)
define void @invariant_store(ptr %dst, ptr readonly %src) {
; CHECK-LABEL: @invariant_store
; CHECK: vector.body:
; CHECK: %[[LOAD1:.*]] = load <vscale x 4 x i32>
; CHECK: %[[LOAD2:.*]] = load <vscale x 4 x i32>
+; CHECK: %[[LOAD3:.*]] = load <vscale x 4 x i32>
+; CHECK: %[[LOAD4:.*]] = load <vscale x 4 x i32>
; CHECK: %[[ADD1:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[LOAD1]]
; CHECK: %[[ADD2:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[LOAD2]]
+; CHECK: %[[ADD3:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[LOAD3]]
+; CHECK: %[[ADD4:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[LOAD4]]
; CHECK: middle.block:
-; CHECK: %[[ADD:.*]] = add <vscale x 4 x i32> %[[ADD2]], %[[ADD1]]
-; CHECK-NEXT: %[[SUM:.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %[[ADD]])
+; CHECK: %[[ADD5:.*]] = add <vscale x 4 x i32> %[[ADD2]], %[[ADD1]]
+; CHECK: %[[ADD6:.*]] = add <vscale x 4 x i32> %[[ADD3]], %[[ADD5]]
+; CHECK: %[[ADD7:.*]] = add <vscale x 4 x i32> %[[ADD4]], %[[ADD6]]
+; CHECK-NEXT: %[[SUM:.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %[[ADD7]])
; CHECK-NEXT: store i32 %[[SUM]], ptr %gep.dst, align 4
entry:
%gep.dst = getelementptr inbounds i32, ptr %dst, i64 42
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll b/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
index b99accfacd8f5..e4c5a9832e139 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
@@ -5,10 +5,10 @@ define i64 @test_vectorize_select_umin_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_umin_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -16,34 +16,58 @@ define i64 @test_vectorize_select_umin_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP26]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp ugt <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ugt <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP4]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP22]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP18]] = select <2 x i1> [[TMP28]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP5]])
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP22]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP22]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP18]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[TMP12]], <2 x i64> [[TMP13]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP9]], 100
; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 0, i64 [[TMP14]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -51,12 +75,12 @@ define i64 @test_vectorize_select_umin_first_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 100, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 100, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
@@ -94,10 +118,10 @@ define i64 @test_vectorize_select_umin_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_umin_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -105,34 +129,58 @@ define i64 @test_vectorize_select_umin_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ splat (i64 100), %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp uge <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP26:%.*]] = icmp uge <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp uge <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP6]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP18]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP8]] = select <2 x i1> [[TMP4]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP16]] = select <2 x i1> [[TMP26]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP18]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <2 x i64> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP18]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[TMP14:%.*]] = select <2 x i1> [[TMP12]], <2 x i64> [[TMP8]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP16]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[TMP13]], <2 x i64> [[TMP14]])
-; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP15]], -9223372036854775808
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP15]], i64 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -140,12 +188,12 @@ define i64 @test_vectorize_select_umin_last_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 100, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 100, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
@@ -183,10 +231,10 @@ define i64 @test_vectorize_select_smin_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_smin_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -194,34 +242,58 @@ define i64 @test_vectorize_select_smin_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP26]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp sgt <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp sgt <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP4]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP22]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP18]] = select <2 x i1> [[TMP28]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP5]])
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP22]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP22]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP18]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[TMP12]], <2 x i64> [[TMP13]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP9]], 0
; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 0, i64 [[TMP14]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -229,12 +301,12 @@ define i64 @test_vectorize_select_smin_first_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
@@ -272,10 +344,10 @@ define i64 @test_vectorize_select_smin_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_smin_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -283,34 +355,58 @@ define i64 @test_vectorize_select_smin_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp sge <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp sge <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP26:%.*]] = icmp sge <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp sge <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP6]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP18]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP8]] = select <2 x i1> [[TMP4]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP16]] = select <2 x i1> [[TMP26]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP18]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <2 x i64> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP18]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[TMP14:%.*]] = select <2 x i1> [[TMP12]], <2 x i64> [[TMP8]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP16]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[TMP13]], <2 x i64> [[TMP14]])
-; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP15]], -9223372036854775808
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP15]], i64 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -318,12 +414,12 @@ define i64 @test_vectorize_select_smin_last_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
@@ -361,10 +457,10 @@ define i64 @test_vectorize_select_umax_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_umax_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -372,34 +468,58 @@ define i64 @test_vectorize_select_umax_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP26]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp ult <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ult <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP4]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP22]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP18]] = select <2 x i1> [[TMP28]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP5]])
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP22]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP22]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP18]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[TMP12]], <2 x i64> [[TMP13]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP9]], 0
; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 0, i64 [[TMP14]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -407,12 +527,12 @@ define i64 @test_vectorize_select_umax_first_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[MIN_VAL]], [[L]]
@@ -450,10 +570,10 @@ define i64 @test_vectorize_select_umax_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_umax_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -461,34 +581,58 @@ define i64 @test_vectorize_select_umax_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP26:%.*]] = icmp ule <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp ule <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP6]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP18]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP8]] = select <2 x i1> [[TMP4]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP16]] = select <2 x i1> [[TMP26]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP18]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <2 x i64> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP18]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[TMP14:%.*]] = select <2 x i1> [[TMP12]], <2 x i64> [[TMP8]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP16]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[TMP13]], <2 x i64> [[TMP14]])
-; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP15]], -9223372036854775808
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP15]], i64 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -496,12 +640,12 @@ define i64 @test_vectorize_select_umax_last_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
@@ -539,10 +683,10 @@ define i64 @test_vectorize_select_smax_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_smax_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -550,34 +694,58 @@ define i64 @test_vectorize_select_smax_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP26]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp slt <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp slt <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP4]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP22]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP18]] = select <2 x i1> [[TMP28]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP5]])
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP22]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP22]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP18]], <2 x i64> splat (i64 -1)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[TMP12]], <2 x i64> [[TMP13]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP9]], 0
; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 0, i64 [[TMP14]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -585,12 +753,12 @@ define i64 @test_vectorize_select_smax_first_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[MIN_VAL]], [[L]]
@@ -628,10 +796,10 @@ define i64 @test_vectorize_select_smax_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_vectorize_select_smax_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -639,34 +807,58 @@ define i64 @test_vectorize_select_smax_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i64> [[STEP_ADD_2]], splat (i64 2)
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[GEP]], i64 2
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[GEP]], i64 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[GEP]], i64 6
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x i64>, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x i64>, ptr [[TMP23]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <2 x i64> [[VEC_PHI2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <2 x i64> [[VEC_PHI3]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP26:%.*]] = icmp sle <2 x i64> [[VEC_PHI6]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp sle <2 x i64> [[VEC_PHI7]], [[WIDE_LOAD10]]
; CHECK-NEXT: [[TMP5]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI2]], <2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP6]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI3]], <2 x i64> [[WIDE_LOAD4]])
+; CHECK-NEXT: [[TMP18]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI6]], <2 x i64> [[WIDE_LOAD9]])
+; CHECK-NEXT: [[TMP19]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC_PHI7]], <2 x i64> [[WIDE_LOAD10]])
; CHECK-NEXT: [[TMP7]] = select <2 x i1> [[TMP3]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP8]] = select <2 x i1> [[TMP4]], <2 x i64> [[STEP_ADD]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
+; CHECK-NEXT: [[TMP16]] = select <2 x i1> [[TMP26]], <2 x i64> [[STEP_ADD_2]], <2 x i64> [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP17]] = select <2 x i1> [[TMP27]], <2 x i64> [[STEP_ADD_3]], <2 x i64> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX11:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX]], <2 x i64> [[TMP18]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX11]], <2 x i64> [[TMP19]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX12]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <2 x i64> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <2 x i64> [[TMP18]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <2 x i64> [[TMP19]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i64> [[TMP7]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[TMP14:%.*]] = select <2 x i1> [[TMP12]], <2 x i64> [[TMP8]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP24:%.*]] = select <2 x i1> [[TMP20]], <2 x i64> [[TMP16]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> [[TMP17]], <2 x i64> splat (i64 -9223372036854775808)
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[TMP13]], <2 x i64> [[TMP14]])
-; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX14:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX5]], <2 x i64> [[TMP24]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[RDX_MINMAX14]], <2 x i64> [[TMP25]])
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[RDX_MINMAX15]])
; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP15]], -9223372036854775808
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP15]], i64 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -674,12 +866,12 @@ define i64 @test_vectorize_select_smax_last_idx(ptr %src, i64 %n) {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
@@ -759,71 +951,146 @@ define i32 @test_multi_use_reduction_with_trunc_iv(ptr %src, i32 %n) {
; CHECK-NEXT: [[PRE:%.*]] = icmp eq i32 [[N]], 0
; CHECK-NEXT: br i1 [[PRE]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
; CHECK: [[LOOP_PREHEADER]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N_EXT]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N_EXT]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_EXT]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N_EXT]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_EXT]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i32> [[STEP_ADD_2]], splat (i32 4)
; CHECK-NEXT: [[IV:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[GEP_SRC]], i64 4
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[GEP_SRC]], i64 8
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[GEP_SRC]], i64 12
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[GEP_SRC]], align 4
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP20]], align 4
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP21]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i32> [[WIDE_LOAD]], [[VEC_PHI2]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i32> [[WIDE_LOAD4]], [[VEC_PHI3]]
+; CHECK-NEXT: [[TMP24:%.*]] = icmp ugt <4 x i32> [[WIDE_LOAD10]], [[VEC_PHI7]]
+; CHECK-NEXT: [[TMP25:%.*]] = icmp ugt <4 x i32> [[WIDE_LOAD11]], [[VEC_PHI8]]
; CHECK-NEXT: [[TMP5]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI2]])
; CHECK-NEXT: [[TMP6]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[WIDE_LOAD4]], <4 x i32> [[VEC_PHI3]])
+; CHECK-NEXT: [[TMP18]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[WIDE_LOAD10]], <4 x i32> [[VEC_PHI7]])
+; CHECK-NEXT: [[TMP19]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[WIDE_LOAD11]], <4 x i32> [[VEC_PHI8]])
; CHECK-NEXT: [[TMP7]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[VEC_IND]]
; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP4]], <4 x i32> [[VEC_PHI1]], <4 x i32> [[STEP_ADD]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-NEXT: [[TMP16]] = select <4 x i1> [[TMP24]], <4 x i32> [[VEC_PHI5]], <4 x i32> [[STEP_ADD_2]]
+; CHECK-NEXT: [[TMP17]] = select <4 x i1> [[TMP25]], <4 x i32> [[VEC_PHI4]], <4 x i32> [[STEP_ADD_3]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[TMP5]], <4 x i32> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[RDX_MINMAX]])
+; CHECK-NEXT: [[RDX_MINMAX12:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[RDX_MINMAX]], <4 x i32> [[TMP18]])
+; CHECK-NEXT: [[RDX_MINMAX13:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[RDX_MINMAX12]], <4 x i32> [[TMP19]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[RDX_MINMAX13]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <4 x i32> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <4 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq <4 x i32> [[TMP18]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP23:%.*]] = icmp eq <4 x i32> [[TMP19]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP7]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP12]], <4 x i32> [[TMP8]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP22]], <4 x i32> [[TMP16]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = select <4 x i1> [[TMP23]], <4 x i32> [[TMP17]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[TMP13]], <4 x i32> [[TMP14]])
-; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[RDX_MINMAX5]])
+; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[RDX_MINMAX5]], <4 x i32> [[TMP26]])
+; CHECK-NEXT: [[RDX_MINMAX16:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[RDX_MINMAX15]], <4 x i32> [[TMP27]])
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[RDX_MINMAX16]])
; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i32 [[TMP15]], 0
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP15]], i32 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ 1, %[[LOOP_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF19:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], 0
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP31]], i32 0, i32 [[BC_MERGE_RDX]]
+; CHECK-NEXT: [[N_MOD_VF18:%.*]] = urem i64 [[N_EXT]], 4
+; CHECK-NEXT: [[N_VEC19:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF18]]
+; CHECK-NEXT: [[TMP33:%.*]] = add i64 1, [[N_VEC19]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT20:%.*]] = insertelement <4 x i32> poison, i32 [[TMP32]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT21:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT20]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT22:%.*]] = insertelement <4 x i32> poison, i32 [[BC_MERGE_RDX17]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT23:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT22]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i32
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT24:%.*]] = insertelement <4 x i32> poison, i32 [[TMP34]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT25:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT24]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT25]], <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX6]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[INDEX26:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT31:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI27:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT21]], %[[VEC_EPILOG_PH]] ], [ [[TMP39:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI28:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT23]], %[[VEC_EPILOG_PH]] ], [ [[TMP38:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND29:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT32:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV1:%.*]] = add i64 1, [[INDEX26]]
; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV1]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD30:%.*]] = load <4 x i32>, ptr [[GEP_SRC1]], align 4
+; CHECK-NEXT: [[TMP37:%.*]] = icmp ugt <4 x i32> [[WIDE_LOAD30]], [[VEC_PHI28]]
+; CHECK-NEXT: [[TMP38]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[WIDE_LOAD30]], <4 x i32> [[VEC_PHI28]])
+; CHECK-NEXT: [[TMP39]] = select <4 x i1> [[TMP37]], <4 x i32> [[VEC_PHI27]], <4 x i32> [[VEC_IND29]]
+; CHECK-NEXT: [[INDEX_NEXT31]] = add nuw i64 [[INDEX26]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT32]] = add <4 x i32> [[VEC_IND29]], splat (i32 4)
+; CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT31]], [[N_VEC19]]
+; CHECK-NEXT: br i1 [[TMP40]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP38]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT33:%.*]] = insertelement <4 x i32> poison, i32 [[TMP41]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT34:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT33]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP42:%.*]] = icmp eq <4 x i32> [[TMP38]], [[BROADCAST_SPLAT34]]
+; CHECK-NEXT: [[TMP43:%.*]] = select <4 x i1> [[TMP42]], <4 x i32> [[TMP39]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP43]])
+; CHECK-NEXT: [[TMP45:%.*]] = icmp ne i32 [[TMP44]], 0
+; CHECK-NEXT: [[TMP46:%.*]] = select i1 [[TMP45]], i32 [[TMP44]], i32 0
+; CHECK-NEXT: [[CMP_N35:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC19]]
+; CHECK-NEXT: br i1 [[CMP_N35]], label %[[EXIT_LOOPEXIT]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL36:%.*]] = phi i64 [ [[TMP33]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP0]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX37:%.*]] = phi i32 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX38:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV2:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL36]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX37]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX38]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[GEP_SRC2:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV2]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC2]], align 4
; CHECK-NEXT: [[C_0:%.*]] = icmp ugt i32 [[L]], [[MIN_VAL]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i32 @llvm.umin.i32(i32 [[L]], i32 [[MIN_VAL]])
-; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
+; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV2]] to i32
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[C_0]], i32 [[MIN_IDX]], i32 [[IV_TRUNC]]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N_EXT]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV2]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV2]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT]], label %[[LOOP1]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
-; CHECK-NEXT: [[MIN_IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MIN_IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
index 9f086d2da37df..25d1107637c37 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
@@ -11,12 +11,12 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP2]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -24,19 +24,33 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 1, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi i64 [ -1, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi i64 [ -1, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi i64 [ -1, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[TMP4]], 3
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP6]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP10]], align 8
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i64>, ptr [[TMP15]], align 8
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 2 x i64>, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 2 x i64>, ptr [[TMP9]], align 8
; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP17]] = and i64 [[VEC_PHI]], [[TMP16]]
; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD3]])
; CHECK-NEXT: [[TMP19]] = and i64 [[VEC_PHI2]], [[TMP18]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP20]] = and i64 [[VEC_PHI3]], [[TMP14]]
+; CHECK-NEXT: [[TMP23:%.*]] = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD7]])
+; CHECK-NEXT: [[TMP21]] = and i64 [[VEC_PHI4]], [[TMP23]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = and i64 [[TMP19]], [[TMP17]]
+; CHECK-NEXT: [[BIN_RDX8:%.*]] = and i64 [[TMP20]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX9:%.*]] = and i64 [[TMP21]], [[BIN_RDX8]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
@@ -44,7 +58,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX9]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[N]], 2
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -63,7 +77,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BIN_RDX9]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -75,7 +89,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: for.end:
-; CHECK-NEXT: [[AND_LCSSA:%.*]] = phi i64 [ [[AND]], [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[AND_LCSSA:%.*]] = phi i64 [ [[AND]], [[FOR_BODY]] ], [ [[BIN_RDX9]], [[MIDDLE_BLOCK]] ], [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[AND_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
index 812408cadfcdb..43657b52a17c9 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
@@ -15,46 +15,82 @@ define i32 @read_only_loop_with_runtime_check(ptr noundef %array, i32 noundef %c
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
; CHECK-NEXT: [[DOTNOT_NOT:%.*]] = icmp ugt i32 [[COUNT]], [[TMP1]]
; CHECK-NEXT: br i1 [[DOTNOT_NOT]], label [[FOR_BODY_PREHEADER10:%.*]], label [[IF_THEN:%.*]]
-; CHECK: for.body.preheader10:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
+; CHECK: iter.check:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER13:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.main.loop.iter.check:
+; CHECK-NEXT: [[MIN_ITERS_CHECK11:%.*]] = icmp ult i32 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK11]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967288
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 12
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967280
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI11:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI11:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI14:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 32
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
; CHECK-NEXT: [[TMP4]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP5]] = add <4 x i32> [[WIDE_LOAD12]], [[VEC_PHI11]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP10]] = add <4 x i32> [[WIDE_LOAD16]], [[VEC_PHI13]]
+; CHECK-NEXT: [[TMP9]] = add <4 x i32> [[WIDE_LOAD17]], [[VEC_PHI14]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP5]], [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX18:%.*]] = add <4 x i32> [[TMP10]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX19:%.*]] = add <4 x i32> [[TMP9]], [[BIN_RDX18]]
+; CHECK-NEXT: [[TMP11:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX19]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER13]]
-; CHECK: for.body.preheader13:
-; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[SUM_07_PH:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK: vec.epilog.iter.check:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[FOR_BODY_PREHEADER13]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: vec.epilog.ph:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; CHECK-NEXT: [[N_VEC21:%.*]] = and i64 [[TMP0]], 4294967292
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: vec.epilog.vector.body:
+; CHECK-NEXT: [[INDEX22:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI23:%.*]] = phi <4 x i32> [ [[TMP12]], [[VEC_EPILOG_PH]] ], [ [[TMP14:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[INDEX22]]
+; CHECK-NEXT: [[WIDE_LOAD24:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4
+; CHECK-NEXT: [[TMP14]] = add <4 x i32> [[WIDE_LOAD24]], [[VEC_PHI23]]
+; CHECK-NEXT: [[INDEX_NEXT25]] = add nuw i64 [[INDEX22]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[N_VEC21]]
+; CHECK-NEXT: br i1 [[TMP15]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: vec.epilog.middle.block:
+; CHECK-NEXT: [[TMP16:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP14]])
+; CHECK-NEXT: [[CMP_N26:%.*]] = icmp eq i64 [[N_VEC21]], [[TMP0]]
+; CHECK-NEXT: br i1 [[CMP_N26]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER13]]
+; CHECK: for.body.preheader28:
+; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC21]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[SUM_07_PH:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ [[ADD:%.*]], [[FOR_BODY1]] ]
; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; CHECK: for.body:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER13]] ]
-; CHECK-NEXT: [[SUM_07:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[SUM_07_PH]], [[FOR_BODY_PREHEADER13]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY1]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER13]] ]
+; CHECK-NEXT: [[SUM_07:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY1]] ], [ [[SUM_07_PH]], [[FOR_BODY_PREHEADER13]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ADD]] = add nsw i32 [[TMP8]], [[SUM_07]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[TMP0]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: if.then:
; CHECK-NEXT: tail call void @llvm.trap()
; CHECK-NEXT: unreachable
@@ -129,47 +165,83 @@ define dso_local noundef i32 @sum_prefix_with_sum(ptr %s.coerce0, i64 %s.coerce1
; CHECK: for.body.preheader:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[DOTNOT_NOT:%.*]] = icmp ugt i64 [[S_COERCE1]], [[TMP0]]
-; CHECK-NEXT: br i1 [[DOTNOT_NOT]], label [[FOR_BODY_PREHEADER8:%.*]], label [[COND_FALSE_I:%.*]], !prof [[PROF4:![0-9]+]]
-; CHECK: for.body.preheader8:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[DOTNOT_NOT]], label [[ITER_CHECK:%.*]], label [[COND_FALSE_I:%.*]], !prof [[PROF6:![0-9]+]]
+; CHECK: iter.check:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER11:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.main.loop.iter.check:
+; CHECK-NEXT: [[MIN_ITERS_CHECK9:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK9]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 12
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -16
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP3:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI11:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP9:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI12:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP8:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[S_COERCE0]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
; CHECK-NEXT: [[TMP3]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP4]] = add <4 x i32> [[WIDE_LOAD10]], [[VEC_PHI9]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP9]] = add <4 x i32> [[WIDE_LOAD14]], [[VEC_PHI11]]
+; CHECK-NEXT: [[TMP8]] = add <4 x i32> [[WIDE_LOAD15]], [[VEC_PHI12]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label [[SPAN_CHECKED_ACCESS_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP4]], [[TMP3]]
-; CHECK-NEXT: [[ADD:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX16:%.*]] = add <4 x i32> [[TMP9]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX17:%.*]] = add <4 x i32> [[TMP8]], [[BIN_RDX16]]
+; CHECK-NEXT: [[TMP10:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX17]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER11]]
-; CHECK: for.body.preheader11:
-; CHECK-NEXT: [[I_07_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER8]] ], [ [[N_VEC]], [[SPAN_CHECKED_ACCESS_EXIT]] ]
-; CHECK-NEXT: [[RET_0_LCSSA:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER8]] ], [ [[ADD]], [[SPAN_CHECKED_ACCESS_EXIT]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK: vec.epilog.iter.check:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[FOR_BODY_PREHEADER11]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK: vec.epilog.ph:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; CHECK-NEXT: [[N_VEC19:%.*]] = and i64 [[N]], -4
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
+; CHECK: vec.epilog.vector.body:
+; CHECK-NEXT: [[INDEX20:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT23:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_PHI21:%.*]] = phi <4 x i32> [ [[TMP11]], [[VEC_EPILOG_PH]] ], [ [[TMP13:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i8], ptr [[S_COERCE0]], i64 [[INDEX20]]
+; CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[TMP13]] = add <4 x i32> [[WIDE_LOAD22]], [[VEC_PHI21]]
+; CHECK-NEXT: [[INDEX_NEXT23]] = add nuw i64 [[INDEX20]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT23]], [[N_VEC19]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: vec.epilog.middle.block:
+; CHECK-NEXT: [[TMP15:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP13]])
+; CHECK-NEXT: [[CMP_N24:%.*]] = icmp eq i64 [[N]], [[N_VEC19]]
+; CHECK-NEXT: br i1 [[CMP_N24]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER11]]
+; CHECK: for.body.preheader26:
+; CHECK-NEXT: [[I_07_PH:%.*]] = phi i64 [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC19]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RET_06_PH:%.*]] = phi i32 [ 0, [[ITER_CHECK]] ], [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label [[FOR_BODY2:%.*]]
; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[RET_0_LCSSA1:%.*]] = phi i32 [ 0, [[ENTRY1:%.*]] ], [ [[ADD]], [[SPAN_CHECKED_ACCESS_EXIT]] ], [ [[ADD1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT: [[RET_0_LCSSA1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[ADD1:%.*]], [[FOR_BODY2]] ]
; CHECK-NEXT: ret i32 [[RET_0_LCSSA1]]
; CHECK: for.body:
-; CHECK-NEXT: [[I_07:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY1]] ], [ [[I_07_PH]], [[FOR_BODY_PREHEADER11]] ]
-; CHECK-NEXT: [[RET_06:%.*]] = phi i32 [ [[ADD1]], [[FOR_BODY1]] ], [ [[RET_0_LCSSA]], [[FOR_BODY_PREHEADER11]] ]
+; CHECK-NEXT: [[I_07:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY2]] ], [ [[I_07_PH]], [[FOR_BODY_PREHEADER11]] ]
+; CHECK-NEXT: [[RET_06:%.*]] = phi i32 [ [[ADD1]], [[FOR_BODY2]] ], [ [[RET_06_PH]], [[FOR_BODY_PREHEADER11]] ]
; CHECK-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds [4 x i8], ptr [[S_COERCE0]], i64 [[I_07]]
; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX_I]], align 4
; CHECK-NEXT: [[ADD1]] = add nsw i32 [[TMP7]], [[RET_06]]
; CHECK-NEXT: [[INC]] = add nuw i64 [[I_07]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY2]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: cond.false.i:
; CHECK-NEXT: tail call void @llvm.trap()
; CHECK-NEXT: unreachable
@@ -228,7 +300,7 @@ define hidden noundef nonnull align 4 dereferenceable(4) ptr @span_checked_acces
; CHECK-NEXT: [[__SIZE__I:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 8
; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[__SIZE__I]], align 8
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[__IDX]], [[TMP0]]
-; CHECK-NEXT: br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]], !prof [[PROF4]]
+; CHECK-NEXT: br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]], !prof [[PROF6]]
; CHECK: cond.false:
; CHECK-NEXT: tail call void @llvm.trap()
; CHECK-NEXT: unreachable
@@ -292,8 +364,11 @@ declare void @llvm.lifetime.end.p0(ptr nocapture)
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[PROF4]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK: [[PROF3]] = !{!"branch_weights", i32 4, i32 12}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[PROF6]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/peel-multiple-unreachable-exits-for-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/peel-multiple-unreachable-exits-for-vectorization.ll
index 4518a5dc6d540..0ecc93c8d1cf5 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/peel-multiple-unreachable-exits-for-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/peel-multiple-unreachable-exits-for-vectorization.ll
@@ -37,36 +37,52 @@ define i64 @sum_2_at_with_int_conversion(ptr %A, ptr %B, i64 %N) {
; CHECK-NEXT: br i1 [[TMP6]], label [[ERROR_I10:%.*]], label [[LOOP_PREHEADER:%.*]]
; CHECK: loop.preheader:
; CHECK-NEXT: [[TMP7:%.*]] = add nuw i64 [[SMAX]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 7
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[LOOP_PREHEADER17:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP7]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP7]], -8
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI14:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI15:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr [8 x i8], ptr [[START_I]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 16
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP8]], i64 32
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP8]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 8
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <2 x i64>, ptr [[TMP9]], align 8
+; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <2 x i64>, ptr [[TMP20]], align 8
+; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <2 x i64>, ptr [[TMP21]], align 8
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [8 x i8], ptr [[START_I1]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 16
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP10]], i64 32
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[TMP10]], i64 48
; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <2 x i64>, ptr [[TMP10]], align 8
; CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <2 x i64>, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <2 x i64>, ptr [[TMP24]], align 8
+; CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <2 x i64>, ptr [[TMP25]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP13:%.*]] = add <2 x i64> [[WIDE_LOAD14]], [[VEC_PHI13]]
+; CHECK-NEXT: [[TMP18:%.*]] = add <2 x i64> [[WIDE_LOAD17]], [[VEC_PHI14]]
+; CHECK-NEXT: [[TMP19:%.*]] = add <2 x i64> [[WIDE_LOAD18]], [[VEC_PHI15]]
; CHECK-NEXT: [[TMP14]] = add <2 x i64> [[TMP12]], [[WIDE_LOAD15]]
; CHECK-NEXT: [[TMP15]] = add <2 x i64> [[TMP13]], [[WIDE_LOAD16]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP22]] = add <2 x i64> [[TMP18]], [[WIDE_LOAD21]]
+; CHECK-NEXT: [[TMP23]] = add <2 x i64> [[TMP19]], [[WIDE_LOAD22]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP15]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX23:%.*]] = add <2 x i64> [[TMP22]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX24:%.*]] = add <2 x i64> [[TMP23]], [[BIN_RDX23]]
+; CHECK-NEXT: [[TMP17:%.*]] = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX24]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP7]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[LOOP_PREHEADER17]]
-; CHECK: loop.preheader17:
+; CHECK: loop.preheader25:
; CHECK-NEXT: [[IV_PH:%.*]] = phi i64 [ 0, [[LOOP_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[SUM_PH:%.*]] = phi i64 [ 0, [[LOOP_PREHEADER]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
@@ -152,42 +168,64 @@ define i64 @sum_3_at_with_int_conversion(ptr %A, ptr %B, ptr %C, i64 %N) {
; CHECK-NEXT: br i1 [[TMP8]], label [[ERROR_I21:%.*]], label [[LOOP_PREHEADER:%.*]]
; CHECK: loop.preheader:
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i64 [[SMAX]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 7
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[LOOP_PREHEADER31:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP10]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP10]], -8
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI25:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI26:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI27:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [8 x i8], ptr [[START_I]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 16
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP11]], i64 32
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP11]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP11]], align 8
; CHECK-NEXT: [[WIDE_LOAD26:%.*]] = load <2 x i64>, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[WIDE_LOAD31:%.*]] = load <2 x i64>, ptr [[TMP27]], align 8
+; CHECK-NEXT: [[WIDE_LOAD32:%.*]] = load <2 x i64>, ptr [[TMP28]], align 8
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr [8 x i8], ptr [[START_I1]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i64 16
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[TMP13]], i64 32
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP13]], i64 48
; CHECK-NEXT: [[WIDE_LOAD27:%.*]] = load <2 x i64>, ptr [[TMP13]], align 8
; CHECK-NEXT: [[WIDE_LOAD28:%.*]] = load <2 x i64>, ptr [[TMP14]], align 8
+; CHECK-NEXT: [[WIDE_LOAD33:%.*]] = load <2 x i64>, ptr [[TMP31]], align 8
+; CHECK-NEXT: [[WIDE_LOAD34:%.*]] = load <2 x i64>, ptr [[TMP32]], align 8
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr [8 x i8], ptr [[START_I12]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 16
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[TMP15]], i64 32
+; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[TMP15]], i64 48
; CHECK-NEXT: [[WIDE_LOAD29:%.*]] = load <2 x i64>, ptr [[TMP15]], align 8
; CHECK-NEXT: [[WIDE_LOAD30:%.*]] = load <2 x i64>, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[WIDE_LOAD37:%.*]] = load <2 x i64>, ptr [[TMP35]], align 8
+; CHECK-NEXT: [[WIDE_LOAD38:%.*]] = load <2 x i64>, ptr [[TMP36]], align 8
; CHECK-NEXT: [[TMP17:%.*]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP18:%.*]] = add <2 x i64> [[WIDE_LOAD26]], [[VEC_PHI25]]
+; CHECK-NEXT: [[TMP25:%.*]] = add <2 x i64> [[WIDE_LOAD31]], [[VEC_PHI26]]
+; CHECK-NEXT: [[TMP26:%.*]] = add <2 x i64> [[WIDE_LOAD32]], [[VEC_PHI27]]
; CHECK-NEXT: [[TMP19:%.*]] = add <2 x i64> [[TMP17]], [[WIDE_LOAD27]]
; CHECK-NEXT: [[TMP20:%.*]] = add <2 x i64> [[TMP18]], [[WIDE_LOAD28]]
+; CHECK-NEXT: [[TMP29:%.*]] = add <2 x i64> [[TMP25]], [[WIDE_LOAD33]]
+; CHECK-NEXT: [[TMP30:%.*]] = add <2 x i64> [[TMP26]], [[WIDE_LOAD34]]
; CHECK-NEXT: [[TMP21]] = add <2 x i64> [[TMP19]], [[WIDE_LOAD29]]
; CHECK-NEXT: [[TMP22]] = add <2 x i64> [[TMP20]], [[WIDE_LOAD30]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP33]] = add <2 x i64> [[TMP29]], [[WIDE_LOAD37]]
+; CHECK-NEXT: [[TMP34]] = add <2 x i64> [[TMP30]], [[WIDE_LOAD38]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP22]], [[TMP21]]
-; CHECK-NEXT: [[TMP24:%.*]] = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX39:%.*]] = add <2 x i64> [[TMP33]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX40:%.*]] = add <2 x i64> [[TMP34]], [[BIN_RDX39]]
+; CHECK-NEXT: [[TMP24:%.*]] = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX40]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP10]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[LOOP_PREHEADER31]]
-; CHECK: loop.preheader31:
+; CHECK: loop.preheader41:
; CHECK-NEXT: [[IV_PH:%.*]] = phi i64 [ 0, [[LOOP_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[SUM_PH:%.*]] = phi i64 [ 0, [[LOOP_PREHEADER]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
More information about the llvm-commits
mailing list