[llvm] [SLP] Inefficient cost-modelling and codegen for reductions with slp-… (PR #197875)
Sushant Gokhale via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 17:53:42 PDT 2026
https://github.com/sushgokh updated https://github.com/llvm/llvm-project/pull/197875
>From 8b456d41af3414fa4600ec66cb05726b9e99674d Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 14 May 2026 23:35:29 -0700
Subject: [PATCH 1/8] [SLP] Inefficient cost-modelling and codegen for
reductions with slp-revec
When revectorizing, starting with reduction, SLP generates slightly inefficient code for reduction.
e.g. In the godbolt link https://godbolt.org/z/ez7KPnxM5,
hor_reduction --> original code
hor_reduction_revec_as_imagined_in_SLP --> revectorized code would look like
Rather than extracting per lane, we can extract original leaf nodes of the reduction, which are sub-vectors, and then perform usual reduction as in non-revectorized code. In the above link,
hor_reduction_ideal_revec --> how the revec code should look like
Extracting subvectors and achieving the reduction result would be better than extracting per lane and achieving the same result.
Change in Transforms/SLPVectorizer/RISCV/complex-loads.ll is side effect of this change. The change with the file seems to better. IR post-SLP before/after patch and llvm-mca analysis of the corresponding codegen can be found below:
IR post-SLP before patch: https://godbolt.org/z/q1b7nW416
IR post-SLP after patch: https://godbolt.org/z/W6vsP4zox
codegen comparison: https://godbolt.org/z/TGzjn3K8T
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 86 ++---
.../SLPVectorizer/RISCV/complex-loads.ll | 314 +++++++++++++-----
.../SLPVectorizer/SystemZ/revec-fix-128169.ll | 19 +-
.../revec-reduced-value-vectorized-later.ll | 3 +-
llvm/test/Transforms/SLPVectorizer/revec.ll | 19 +-
5 files changed, 304 insertions(+), 137 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 99ed09cc684b9..87e53f57a2d9b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29044,6 +29044,7 @@ class HorizontalReduction {
// Also, do not try to reduce const values, if the operation is not
// foldable.
bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
+ R.isAnalyzedReductionRoot(EdgeInst) ||
getRdxKind(EdgeInst) != RdxKind ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
ReductionOrdering CurrentRK = IsReducedVal
@@ -29860,10 +29861,11 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered)
return VectorizedTree;
- if (!VectorValuesAndScales.empty())
+ if (!VectorValuesAndScales.empty()) {
VectorizedTree = GetNewVectorizedTree(
VectorizedTree,
- emitReduction(Builder, *TTI, ReductionRoot->getType()));
+ emitReduction(V, Builder, *TTI, ReductionRoot->getType()));
+ }
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@@ -30252,8 +30254,9 @@ class HorizontalReduction {
private:
/// Creates the reduction from the given \p Vec vector value with the given
/// scale \p Scale and signedness \p IsSigned.
- Value *createSingleOp(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Value *Vec, unsigned Scale, bool IsSigned, Type *DestTy,
+ Value *createSingleOp(BoUpSLP &V, IRBuilderBase &Builder,
+ const TargetTransformInfo &TTI, Value *Vec,
+ unsigned Scale, bool IsSigned, Type *DestTy,
bool ReducedInTree) {
Value *Rdx;
if (ReducedInTree) {
@@ -30261,25 +30264,38 @@ class HorizontalReduction {
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
- Rdx = PoisonValue::get(
- getWidenedType(Vec->getType()->getScalarType(), DestTyNumElements));
- for (unsigned I : seq<unsigned>(DestTyNumElements)) {
- // Do reduction for each lane.
- // e.g., do reduce add for
- // VL[0] = <4 x Ty> <a, b, c, d>
- // VL[1] = <4 x Ty> <e, f, g, h>
- // Lane[0] = <2 x Ty> <a, e>
- // Lane[1] = <2 x Ty> <b, f>
- // Lane[2] = <2 x Ty> <c, g>
- // Lane[3] = <2 x Ty> <d, h>
- // result[0] = reduce add Lane[0]
- // result[1] = reduce add Lane[1]
- // result[2] = reduce add Lane[2]
- // result[3] = reduce add Lane[3]
- SmallVector<int, 16> Mask = createStrideMask(I, DestTyNumElements, VF);
- Value *Lane = Builder.CreateShuffleVector(Vec, Mask);
- Rdx = Builder.CreateInsertElement(
- Rdx, emitReduction(Lane, Builder, &TTI, DestTy), I);
+ unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
+ Rdx = nullptr;
+ /*
+ e.g. Consider vector reduce add.
+
+ Initial reduction is
+ %add0 = add <4 x i32> zeroinitializer, %A
+ %add1 = add <4 x i32> %add0, %B
+
+ where,
+ %A = <a, b, c, d>
+ %B = <e, f, g, h>
+ %add1 = A + B = <a + e, b + f, c + g, d + h>
+
+ After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
+ the reduction can be expressed as:
+ %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %add0 = add <4 x i32> zeroinitializer, %A
+ %add1 = add <4 x i32> %add0, %B
+ */
+ for (auto I : seq<unsigned>(VF)) {
+ auto Position = I * DestTyNumElements;
+ Value *SubVec =
+ createExtractVector(Builder, Vec, DestTyNumElements, Position);
+ if (!Rdx) {
+ Rdx = SubVec;
+ } else {
+ Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
+ if (isa<Instruction>(Rdx))
+ V.analyzedReductionRoot(cast<Instruction>(Rdx));
+ }
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
@@ -30379,19 +30395,15 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
+ auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
+ ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
- VectorCost += TTI->getShuffleCost(
- TTI::SK_PermuteSingleSrc,
- FixedVectorType::get(VecTy->getScalarType(),
- ReducedVals.size()),
- VectorTy,
- createStrideMask(I, ScalarTyNumElements, ReducedVals.size()));
- VectorCost += TTI->getArithmeticReductionCost(RdxOpcode, VecTy,
- FMF, CostKind);
+ VectorCost += TTI->getShuffleCost(TTI::SK_ExtractSubvector, VecTy,
+ DstTy, {}, CostKind,
+ I * ScalarTyNumElements, VecTy);
+ VectorCost +=
+ TTI->getArithmeticInstrCost(RdxOpcode, VecTy, CostKind);
}
- VectorCost += TTI->getScalarizationOverhead(
- VecTy, APInt::getAllOnes(ScalarTyNumElements), /*Insert*/ true,
- /*Extract*/ false, TTI::TCK_RecipThroughput);
} else {
Type *RedTy = VectorTy->getElementType();
auto [RType, IsSigned] = R.getRootNodeTypeWithNoCast().value_or(
@@ -30515,13 +30527,13 @@ class HorizontalReduction {
/// Splits the values, stored in VectorValuesAndScales, into registers/free
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
- Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Type *DestTy) {
+ Value *emitReduction(BoUpSLP &V, IRBuilderBase &Builder,
+ const TargetTransformInfo &TTI, Type *DestTy) {
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
bool ReducedInTree) {
- Value *Rdx = createSingleOp(Builder, TTI, Vec, Scale, IsSigned, DestTy,
+ Value *Rdx = createSingleOp(V, Builder, TTI, Vec, Scale, IsSigned, DestTy,
ReducedInTree);
if (ReducedSubTree)
ReducedSubTree = createOp(Builder, RdxKind, ReducedSubTree, Rdx,
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index 09be099fcf709..ed3f85d31d314 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -277,102 +277,265 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: entry:
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20:%.*]] = getelementptr i8, ptr [[PIX1]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22:%.*]] = getelementptr i8, ptr [[PIX2]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25:%.*]] = getelementptr i8, ptr [[PIX1]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27:%.*]] = getelementptr i8, ptr [[PIX2]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34:%.*]] = getelementptr i8, ptr [[PIX2]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37:%.*]] = getelementptr i8, ptr [[PIX1]], i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39:%.*]] = getelementptr i8, ptr [[PIX2]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR644:%.*]] = getelementptr i8, ptr [[PIX2]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR_1:%.*]] = getelementptr i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_4:%.*]] = getelementptr i8, ptr null, i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_3:%.*]] = getelementptr i8, ptr null, i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_3:%.*]] = getelementptr i8, ptr null, i64 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP57:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_3:%.*]] = getelementptr i8, ptr null, i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_3:%.*]] = getelementptr i8, ptr null, i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_3:%.*]] = getelementptr i8, ptr null, i64 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP61:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_3:%.*]] = getelementptr i8, ptr null, i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_3:%.*]] = getelementptr i8, ptr null, i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_3:%.*]] = getelementptr i8, ptr null, i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_3:%.*]] = getelementptr i8, ptr null, i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX37]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX25]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX13]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP60:%.*]] = load i8, ptr [[ARRAYIDX32_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX20_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX8_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = load i8, ptr [[ADD_PTR3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX34_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX22_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX10_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = load i8, ptr [[ADD_PTR644]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX34]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX22]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load i8, ptr [[ARRAYIDX10]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load i8, ptr [[PIX2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX37_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = load i8, ptr [[ARRAYIDX25_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = load i8, ptr [[ARRAYIDX13_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX3_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX39_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX15_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX5_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX39]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX15]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP32:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV_1:%.*]] = zext i8 [[TMP9]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV_3:%.*]] = zext i8 [[TMP49]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV_2:%.*]] = zext i8 [[TMP33]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9_1:%.*]] = zext i8 [[TMP8]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9_3:%.*]] = zext i8 [[TMP48]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9_2:%.*]] = zext i8 [[TMP32]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21_1:%.*]] = zext i8 [[TMP7]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21_3:%.*]] = zext i8 [[TMP47]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21_2:%.*]] = zext i8 [[TMP31]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[PIX1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[PIX2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[TMP3]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = sub <4 x i32> [[TMP6]], [[TMP7]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = sub <4 x i32> [[TMP9]], [[TMP10]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = shl <4 x i32> [[TMP11]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP8]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load <4 x i8>, ptr [[ADD_PTR3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load <4 x i8>, ptr [[ADD_PTR644]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = zext <4 x i8> [[TMP14]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = zext <4 x i8> [[TMP15]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = sub <4 x i32> [[TMP18]], [[TMP19]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = zext <4 x i8> [[TMP16]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = zext <4 x i8> [[TMP17]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = sub <4 x i32> [[TMP21]], [[TMP22]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = shl <4 x i32> [[TMP23]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = add <4 x i32> [[TMP24]], [[TMP20]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load <4 x i8>, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = zext <4 x i8> [[TMP26]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load <4 x i8>, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = zext <4 x i8> [[TMP28]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = sub <4 x i32> [[TMP27]], [[TMP29]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = load i8, ptr [[ARRAYIDX20]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = load i8, ptr [[ARRAYIDX8]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33:%.*]] = zext i8 [[TMP50]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21:%.*]] = zext i8 [[TMP52]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9:%.*]] = zext i8 [[TMP53]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33_1:%.*]] = zext i8 [[TMP60]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33_3:%.*]] = zext i8 [[TMP46]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33_2:%.*]] = zext i8 [[TMP30]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP106:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2:%.*]] = zext i8 [[TMP17]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2_1:%.*]] = zext i8 [[TMP13]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2_3:%.*]] = zext i8 [[TMP106]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2_2:%.*]] = zext i8 [[TMP37]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11:%.*]] = zext i8 [[TMP16]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11_1:%.*]] = zext i8 [[TMP12]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11_3:%.*]] = zext i8 [[TMP56]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11_2:%.*]] = zext i8 [[TMP36]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23:%.*]] = zext i8 [[TMP15]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23_1:%.*]] = zext i8 [[TMP11]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23_3:%.*]] = zext i8 [[TMP55]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23_2:%.*]] = zext i8 [[TMP35]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35:%.*]] = zext i8 [[TMP14]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35_1:%.*]] = zext i8 [[TMP10]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35_3:%.*]] = zext i8 [[TMP54]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35_2:%.*]] = zext i8 [[TMP34]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[TMP59:%.*]] = zext i8 [[TMP58]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[SUB:%.*]] = sub i32 [[TMP59]], [[CONV2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB_1:%.*]] = sub i32 [[CONV_1]], [[CONV2_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB_3:%.*]] = sub i32 [[CONV_3]], [[CONV2_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB_2:%.*]] = sub i32 [[CONV_2]], [[CONV2_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12:%.*]] = sub i32 [[CONV9]], [[CONV11]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12_1:%.*]] = sub i32 [[CONV9_1]], [[CONV11_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12_3:%.*]] = sub i32 [[CONV9_3]], [[CONV11_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12_2:%.*]] = sub i32 [[CONV9_2]], [[CONV11_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24:%.*]] = sub i32 [[CONV21]], [[CONV23]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24_1:%.*]] = sub i32 [[CONV21_1]], [[CONV23_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24_3:%.*]] = sub i32 [[CONV21_3]], [[CONV23_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24_2:%.*]] = sub i32 [[CONV21_2]], [[CONV23_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36:%.*]] = sub i32 [[CONV33]], [[CONV35]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36_1:%.*]] = sub i32 [[CONV33_1]], [[CONV35_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36_3:%.*]] = sub i32 [[CONV33_3]], [[CONV35_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36_2:%.*]] = sub i32 [[CONV33_2]], [[CONV35_2]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP61]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP102:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV4:%.*]] = zext i8 [[TMP5]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV4_1:%.*]] = zext i8 [[TMP21]] to i32
; UNALIGNED_VEC_MEM-NEXT: [[CONV40_2:%.*]] = zext i8 [[TMP42]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = insertelement <2 x i8> poison, i8 [[TMP106]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = insertelement <2 x i8> [[TMP33]], i8 [[TMP57]], i32 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_4]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = zext <4 x i8> [[TMP35]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> poison, i32 [[CONV40_2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = zext <2 x i8> [[TMP34]] to <2 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = shufflevector <2 x i32> [[TMP38]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = shufflevector <4 x i32> [[TMP37]], <4 x i32> [[TMP39]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[CONV38_3]], i32 3
-; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = sub <4 x i32> [[TMP41]], [[TMP36]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = shl <4 x i32> [[TMP58]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = add <4 x i32> [[TMP43]], [[TMP30]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load <4 x i8>, ptr [[ADD_PTR_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = zext <4 x i8> [[TMP45]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = zext <4 x i8> [[TMP46]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = sub <4 x i32> [[TMP49]], [[TMP50]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = zext <4 x i8> [[TMP47]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = zext <4 x i8> [[TMP48]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = sub <4 x i32> [[TMP52]], [[TMP53]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = shl <4 x i32> [[TMP54]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = add <4 x i32> [[TMP55]], [[TMP51]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV4_2:%.*]] = zext i8 [[TMP41]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14:%.*]] = zext i8 [[TMP4]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14_1:%.*]] = zext i8 [[TMP20]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14_3:%.*]] = zext i8 [[TMP57]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14_2:%.*]] = zext i8 [[TMP40]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26:%.*]] = zext i8 [[TMP3]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26_1:%.*]] = zext i8 [[TMP19]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26_3:%.*]] = zext i8 [[TMP61]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26_2:%.*]] = zext i8 [[TMP39]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38:%.*]] = zext i8 [[TMP51]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_1:%.*]] = zext i8 [[TMP18]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP102]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_2:%.*]] = zext i8 [[TMP38]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP62:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP63:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP64:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP65:%.*]] = load i8, ptr [[ARRAYIDX5_4]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6:%.*]] = zext i8 [[TMP29]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6_1:%.*]] = zext i8 [[TMP25]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6_3:%.*]] = zext i8 [[TMP65]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6_2:%.*]] = zext i8 [[TMP45]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16:%.*]] = zext i8 [[TMP28]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16_1:%.*]] = zext i8 [[TMP24]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16_3:%.*]] = zext i8 [[TMP64]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16_2:%.*]] = zext i8 [[TMP44]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28:%.*]] = zext i8 [[TMP27]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28_1:%.*]] = zext i8 [[TMP23]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28_3:%.*]] = zext i8 [[TMP63]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28_2:%.*]] = zext i8 [[TMP43]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40:%.*]] = zext i8 [[TMP26]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40_1:%.*]] = zext i8 [[TMP22]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40_3:%.*]] = zext i8 [[TMP62]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40_4:%.*]] = zext i8 [[TMP78]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7:%.*]] = sub i32 [[CONV4]], [[CONV6]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7_1:%.*]] = sub i32 [[CONV4_1]], [[CONV6_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7_3:%.*]] = sub i32 [[CONV40_2]], [[CONV6_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7_2:%.*]] = sub i32 [[CONV4_2]], [[CONV6_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17:%.*]] = sub i32 [[CONV14]], [[CONV16]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17_1:%.*]] = sub i32 [[CONV14_1]], [[CONV16_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17_3:%.*]] = sub i32 [[CONV14_3]], [[CONV16_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17_2:%.*]] = sub i32 [[CONV14_2]], [[CONV16_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29:%.*]] = sub i32 [[CONV26]], [[CONV28]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29_1:%.*]] = sub i32 [[CONV26_1]], [[CONV28_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29_3:%.*]] = sub i32 [[CONV26_3]], [[CONV28_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29_2:%.*]] = sub i32 [[CONV26_2]], [[CONV28_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41:%.*]] = sub i32 [[CONV38]], [[CONV40]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41_1:%.*]] = sub i32 [[CONV38_1]], [[CONV40_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41_3:%.*]] = sub i32 [[CONV38_3]], [[CONV40_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41_2:%.*]] = sub i32 [[CONV38_2]], [[CONV40_4]]
+; UNALIGNED_VEC_MEM-NEXT: [[SHL:%.*]] = shl i32 [[SUB7]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL_1:%.*]] = shl i32 [[SUB7_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL_3:%.*]] = shl i32 [[SUB7_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL_2:%.*]] = shl i32 [[SUB7_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18:%.*]] = shl i32 [[SUB17]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18_1:%.*]] = shl i32 [[SUB17_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18_3:%.*]] = shl i32 [[SUB17_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18_2:%.*]] = shl i32 [[SUB17_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30:%.*]] = shl i32 [[SUB29]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30_1:%.*]] = shl i32 [[SUB29_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30_3:%.*]] = shl i32 [[SUB29_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30_2:%.*]] = shl i32 [[SUB29_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42:%.*]] = shl i32 [[SUB41]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42_1:%.*]] = shl i32 [[SUB41_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42_3:%.*]] = shl i32 [[SUB41_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42_2:%.*]] = shl i32 [[SUB41_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = add i32 [[SHL]], [[SUB]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = add i32 [[SHL_1]], [[SUB_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = add i32 [[SHL_3]], [[SUB_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = add i32 [[SHL_2]], [[SUB_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = add i32 [[SHL18]], [[SUB12]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = add i32 [[SHL18_1]], [[SUB12_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = add i32 [[SHL18_3]], [[SUB12_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = add i32 [[SHL18_2]], [[SUB12_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = add i32 [[SHL30]], [[SUB24]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = add i32 [[SHL30_1]], [[SUB24_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = add i32 [[SHL30_3]], [[SUB24_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = add i32 [[SHL30_2]], [[SUB24_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = add i32 [[SHL42]], [[SUB36]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = add i32 [[SHL42_1]], [[SUB36_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = add i32 [[SHL42_3]], [[SUB36_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = add i32 [[SHL42_2]], [[SUB36_2]]
; UNALIGNED_VEC_MEM-NEXT: [[SUB45:%.*]] = sub i32 [[ADD_3]], [[ADD31_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_1:%.*]] = sub i32 [[ADD_2]], [[ADD31_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_3:%.*]] = sub i32 [[ADD19_3]], [[ADD43_4]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_2:%.*]] = sub i32 [[ADD19_2]], [[ADD43_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44:%.*]] = add i32 [[ADD31_3]], [[ADD_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_1:%.*]] = add i32 [[ADD31_2]], [[ADD_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_3:%.*]] = add i32 [[ADD43_4]], [[ADD19_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_2:%.*]] = add i32 [[ADD43_2]], [[ADD19_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47:%.*]] = sub i32 [[ADD31]], [[ADD43]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_1:%.*]] = sub i32 [[ADD31_1]], [[ADD43_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_3:%.*]] = sub i32 [[ADD31_4]], [[ADD43_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_2:%.*]] = sub i32 [[ADD31_5]], [[ADD43_5]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46:%.*]] = add i32 [[ADD43]], [[ADD31]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46_1:%.*]] = add i32 [[ADD43_1]], [[ADD31_1]]
@@ -427,17 +590,18 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: [[ADD103:%.*]] = add i32 [[ADD94]], [[ADD78]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD105:%.*]] = add i32 [[SUB102]], [[SUB86]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP66:%.*]] = insertelement <16 x i32> poison, i32 [[ADD46_2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP74]], <16 x i32> <i32 0, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = insertelement <16 x i32> [[TMP66]], i32 [[CONV_3]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP68:%.*]] = insertelement <16 x i32> [[TMP67]], i32 [[ADD46_1]], i32 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP69:%.*]] = insertelement <16 x i32> [[TMP68]], i32 [[ADD46]], i32 3
-; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = shufflevector <4 x i32> [[TMP49]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP69]], <16 x i32> [[TMP78]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP70:%.*]] = insertelement <16 x i32> [[TMP69]], i32 [[CONV_2]], i32 4
+; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = insertelement <16 x i32> [[TMP70]], i32 [[CONV9_2]], i32 5
; UNALIGNED_VEC_MEM-NEXT: [[TMP72:%.*]] = insertelement <16 x i32> [[TMP71]], i32 [[SUB47_1]], i32 6
; UNALIGNED_VEC_MEM-NEXT: [[TMP73:%.*]] = insertelement <16 x i32> [[TMP72]], i32 [[SUB47]], i32 7
-; UNALIGNED_VEC_MEM-NEXT: [[TMP118:%.*]] = shufflevector <4 x i32> [[TMP18]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = shufflevector <16 x i32> [[TMP73]], <16 x i32> [[TMP118]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = insertelement <16 x i32> [[TMP73]], i32 [[CONV_1]], i32 8
+; UNALIGNED_VEC_MEM-NEXT: [[TMP75:%.*]] = insertelement <16 x i32> [[TMP74]], i32 [[CONV9_1]], i32 9
+; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = insertelement <16 x i32> [[TMP75]], i32 [[CONV21_1]], i32 10
; UNALIGNED_VEC_MEM-NEXT: [[TMP77:%.*]] = insertelement <16 x i32> [[TMP76]], i32 [[ADD44]], i32 11
+; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
; UNALIGNED_VEC_MEM-NEXT: [[TMP79:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP80:%.*]] = shufflevector <16 x i32> [[TMP77]], <16 x i32> [[TMP79]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
; UNALIGNED_VEC_MEM-NEXT: [[TMP81:%.*]] = lshr <16 x i32> [[TMP80]], splat (i32 15)
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
index 3f4436f33fad6..274d9a6e950a5 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
@@ -38,18 +38,13 @@ define void @e(<4 x i16> %0) {
; THRESH-NEXT: [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP9:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; THRESH-NEXT: [[TMP13:%.*]] = icmp sgt <16 x i16> [[TMP9]], zeroinitializer
-; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; THRESH-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
-; THRESH-NEXT: [[TMP23:%.*]] = insertelement <4 x i1> poison, i1 [[TMP15]], i64 0
-; THRESH-NEXT: [[TMP16:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; THRESH-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
-; THRESH-NEXT: [[TMP24:%.*]] = insertelement <4 x i1> [[TMP23]], i1 [[TMP17]], i64 1
-; THRESH-NEXT: [[TMP18:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; THRESH-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
-; THRESH-NEXT: [[TMP22:%.*]] = insertelement <4 x i1> [[TMP24]], i1 [[TMP19]], i64 2
-; THRESH-NEXT: [[TMP20:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; THRESH-NEXT: [[TMP21:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP20]])
-; THRESH-NEXT: [[TMP25:%.*]] = insertelement <4 x i1> [[TMP22]], i1 [[TMP21]], i64 3
+; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; THRESH-NEXT: [[TMP11:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; THRESH-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP14]], [[TMP11]]
+; THRESH-NEXT: [[TMP12:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; THRESH-NEXT: [[RDX_OP1:%.*]] = or <4 x i1> [[RDX_OP]], [[TMP12]]
+; THRESH-NEXT: [[TMP15:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; THRESH-NEXT: [[TMP25:%.*]] = or <4 x i1> [[RDX_OP1]], [[TMP15]]
; THRESH-NEXT: [[TMP26]] = zext <4 x i1> [[TMP25]] to <4 x i32>
; THRESH-NEXT: br label [[VECTOR_BODY]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 4305ad171c4b9..3fb5fd9c6b5dc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -4,7 +4,8 @@
define <4 x i16> @test() {
; CHECK-LABEL: define <4 x i16> @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: ret <4 x i16> zeroinitializer
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, zeroinitializer
+; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
%subi = add <4 x i16> zeroinitializer, zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 66507f647a828..2849c82142b55 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -156,18 +156,13 @@ define <4 x i1> @test6(ptr %in1, ptr %in2) {
; CHECK-NEXT: [[TMP22:%.*]] = and <16 x i1> [[TMP11]], [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <32 x i1> [[TMP6]], <32 x i1> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[TMP24:%.*]] = and <16 x i1> [[TMP22]], [[TMP23]]
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
-; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i1> poison, i1 [[TMP26]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i1> [[TMP27]], i1 [[TMP29]], i64 1
-; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[TMP32:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP31]])
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i1> [[TMP30]], i1 [[TMP32]], i64 2
-; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: [[TMP35:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP34]])
-; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i1> [[TMP33]], i1 [[TMP35]], i64 3
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP15]], [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = or <4 x i1> [[RDX_OP]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP36:%.*]] = or <4 x i1> [[RDX_OP1]], [[TMP14]]
; CHECK-NEXT: [[VBSL:%.*]] = select <4 x i1> [[TMP36]], <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: [[CMP:%.*]] = icmp ugt <4 x i32> [[VBSL]], <i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: ret <4 x i1> [[CMP]]
>From be589da7ee3e3e83b31731ac68e1b8d07d6cee02 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Mon, 18 May 2026 22:12:43 -0700
Subject: [PATCH 2/8] Address regression with
Transforms/SLPVectorizer/RISCV/complex-loads.ll
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 52 ++-
.../SLPVectorizer/RISCV/complex-loads.ll | 314 +++++-------------
.../revec-reduced-value-vectorized-later.ll | 3 +-
3 files changed, 99 insertions(+), 270 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 87e53f57a2d9b..0e84153ad6e5e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29044,7 +29044,6 @@ class HorizontalReduction {
// Also, do not try to reduce const values, if the operation is not
// foldable.
bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
- R.isAnalyzedReductionRoot(EdgeInst) ||
getRdxKind(EdgeInst) != RdxKind ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
ReductionOrdering CurrentRK = IsReducedVal
@@ -29861,11 +29860,10 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered)
return VectorizedTree;
- if (!VectorValuesAndScales.empty()) {
+ if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
VectorizedTree,
- emitReduction(V, Builder, *TTI, ReductionRoot->getType()));
- }
+ emitReduction(Builder, *TTI, ReductionRoot->getType()));
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@@ -30254,9 +30252,8 @@ class HorizontalReduction {
private:
/// Creates the reduction from the given \p Vec vector value with the given
/// scale \p Scale and signedness \p IsSigned.
- Value *createSingleOp(BoUpSLP &V, IRBuilderBase &Builder,
- const TargetTransformInfo &TTI, Value *Vec,
- unsigned Scale, bool IsSigned, Type *DestTy,
+ Value *createSingleOp(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
+ Value *Vec, unsigned Scale, bool IsSigned, Type *DestTy,
bool ReducedInTree) {
Value *Rdx;
if (ReducedInTree) {
@@ -30264,26 +30261,20 @@ class HorizontalReduction {
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
- unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
Rdx = nullptr;
/*
e.g. Consider vector reduce add.
- Initial reduction is
- %add0 = add <4 x i32> zeroinitializer, %A
- %add1 = add <4 x i32> %add0, %B
-
- where,
- %A = <a, b, c, d>
- %B = <e, f, g, h>
- %add1 = A + B = <a + e, b + f, c + g, d + h>
+ RdxVal[0] = [a, b, c, d]
+ RdxVal[1] = [e, f, g, h]
+ Add0 = zeroinitializer + RdxVal[0]
+ Add1 = Add0 + RdxVal[1]
- After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
+ After revectorization with VF=2 and Vec = [a, b, c, d, e, f, g, h],
the reduction can be expressed as:
- %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
- %add0 = add <4 x i32> zeroinitializer, %A
- %add1 = add <4 x i32> %add0, %B
+ RdxVal[0] = ExtractVector(Vec, 0, 3) = [a, b, c, d]
+ RdxVal[1] = ExtractVector(Vec, 4, 7) = [e, f, g, h]
+ Add = RdxVal[0] + RdxVal[1]
*/
for (auto I : seq<unsigned>(VF)) {
auto Position = I * DestTyNumElements;
@@ -30293,8 +30284,6 @@ class HorizontalReduction {
Rdx = SubVec;
} else {
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
- if (isa<Instruction>(Rdx))
- V.analyzedReductionRoot(cast<Instruction>(Rdx));
}
}
} else {
@@ -30397,10 +30386,10 @@ class HorizontalReduction {
unsigned ScalarTyNumElements = VecTy->getNumElements();
auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
ReducedVals.size());
- for (unsigned I : seq<unsigned>(ReducedVals.size())) {
- VectorCost += TTI->getShuffleCost(TTI::SK_ExtractSubvector, VecTy,
- DstTy, {}, CostKind,
- I * ScalarTyNumElements, VecTy);
+ for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
+ VectorCost +=
+ ::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
+ CostKind, I * ScalarTyNumElements, VecTy);
VectorCost +=
TTI->getArithmeticInstrCost(RdxOpcode, VecTy, CostKind);
}
@@ -30527,13 +30516,13 @@ class HorizontalReduction {
/// Splits the values, stored in VectorValuesAndScales, into registers/free
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
- Value *emitReduction(BoUpSLP &V, IRBuilderBase &Builder,
- const TargetTransformInfo &TTI, Type *DestTy) {
+ Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
+ Type *DestTy) {
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
bool ReducedInTree) {
- Value *Rdx = createSingleOp(V, Builder, TTI, Vec, Scale, IsSigned, DestTy,
+ Value *Rdx = createSingleOp(Builder, TTI, Vec, Scale, IsSigned, DestTy,
ReducedInTree);
if (ReducedSubTree)
ReducedSubTree = createOp(Builder, RdxKind, ReducedSubTree, Rdx,
@@ -31201,6 +31190,9 @@ bool SLPVectorizerPass::vectorizeHorReduction(
Res = true;
if (auto *I = dyn_cast<Instruction>(VectorizedV); I && I != Inst) {
// Try to find another reduction.
+ if (SLPReVec && I->getType()->isVectorTy() &&
+ Inst->getType()->isVectorTy())
+ continue;
Stack.emplace(I, Level);
continue;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index ed3f85d31d314..09be099fcf709 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -277,265 +277,102 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: entry:
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20:%.*]] = getelementptr i8, ptr [[PIX1]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22:%.*]] = getelementptr i8, ptr [[PIX2]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25:%.*]] = getelementptr i8, ptr [[PIX1]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27:%.*]] = getelementptr i8, ptr [[PIX2]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34:%.*]] = getelementptr i8, ptr [[PIX2]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37:%.*]] = getelementptr i8, ptr [[PIX1]], i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39:%.*]] = getelementptr i8, ptr [[PIX2]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR644:%.*]] = getelementptr i8, ptr [[PIX2]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR_1:%.*]] = getelementptr i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_4:%.*]] = getelementptr i8, ptr null, i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_3:%.*]] = getelementptr i8, ptr null, i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_3:%.*]] = getelementptr i8, ptr null, i64 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP57:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_3:%.*]] = getelementptr i8, ptr null, i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_3:%.*]] = getelementptr i8, ptr null, i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_3:%.*]] = getelementptr i8, ptr null, i64 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP61:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_3:%.*]] = getelementptr i8, ptr null, i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_3:%.*]] = getelementptr i8, ptr null, i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_3:%.*]] = getelementptr i8, ptr null, i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_3:%.*]] = getelementptr i8, ptr null, i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX37]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX25]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX13]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP60:%.*]] = load i8, ptr [[ARRAYIDX32_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX20_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX8_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = load i8, ptr [[ADD_PTR3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX34_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX22_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX10_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = load i8, ptr [[ADD_PTR644]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX34]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX22]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load i8, ptr [[ARRAYIDX10]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load i8, ptr [[PIX2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX37_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = load i8, ptr [[ARRAYIDX25_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = load i8, ptr [[ARRAYIDX13_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX3_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX39_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX15_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX5_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX39]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX15]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP32:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV_1:%.*]] = zext i8 [[TMP9]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV_3:%.*]] = zext i8 [[TMP49]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV_2:%.*]] = zext i8 [[TMP33]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9_1:%.*]] = zext i8 [[TMP8]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9_3:%.*]] = zext i8 [[TMP48]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9_2:%.*]] = zext i8 [[TMP32]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21_1:%.*]] = zext i8 [[TMP7]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21_3:%.*]] = zext i8 [[TMP47]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21_2:%.*]] = zext i8 [[TMP31]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[PIX1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = load i8, ptr [[ARRAYIDX20]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = load i8, ptr [[ARRAYIDX8]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33:%.*]] = zext i8 [[TMP50]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21:%.*]] = zext i8 [[TMP52]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9:%.*]] = zext i8 [[TMP53]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33_1:%.*]] = zext i8 [[TMP60]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33_3:%.*]] = zext i8 [[TMP46]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33_2:%.*]] = zext i8 [[TMP30]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[PIX2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[TMP3]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = sub <4 x i32> [[TMP6]], [[TMP7]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = sub <4 x i32> [[TMP9]], [[TMP10]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = shl <4 x i32> [[TMP11]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP8]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load <4 x i8>, ptr [[ADD_PTR3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load <4 x i8>, ptr [[ADD_PTR644]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = zext <4 x i8> [[TMP14]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = zext <4 x i8> [[TMP15]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = sub <4 x i32> [[TMP18]], [[TMP19]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = zext <4 x i8> [[TMP16]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = zext <4 x i8> [[TMP17]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = sub <4 x i32> [[TMP21]], [[TMP22]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = shl <4 x i32> [[TMP23]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = add <4 x i32> [[TMP24]], [[TMP20]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load <4 x i8>, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = zext <4 x i8> [[TMP26]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load <4 x i8>, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = zext <4 x i8> [[TMP28]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = sub <4 x i32> [[TMP27]], [[TMP29]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP106:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2:%.*]] = zext i8 [[TMP17]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2_1:%.*]] = zext i8 [[TMP13]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2_3:%.*]] = zext i8 [[TMP106]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2_2:%.*]] = zext i8 [[TMP37]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11:%.*]] = zext i8 [[TMP16]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11_1:%.*]] = zext i8 [[TMP12]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11_3:%.*]] = zext i8 [[TMP56]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11_2:%.*]] = zext i8 [[TMP36]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23:%.*]] = zext i8 [[TMP15]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23_1:%.*]] = zext i8 [[TMP11]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23_3:%.*]] = zext i8 [[TMP55]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23_2:%.*]] = zext i8 [[TMP35]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35:%.*]] = zext i8 [[TMP14]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35_1:%.*]] = zext i8 [[TMP10]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35_3:%.*]] = zext i8 [[TMP54]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35_2:%.*]] = zext i8 [[TMP34]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP59:%.*]] = zext i8 [[TMP58]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[SUB:%.*]] = sub i32 [[TMP59]], [[CONV2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB_1:%.*]] = sub i32 [[CONV_1]], [[CONV2_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB_3:%.*]] = sub i32 [[CONV_3]], [[CONV2_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB_2:%.*]] = sub i32 [[CONV_2]], [[CONV2_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12:%.*]] = sub i32 [[CONV9]], [[CONV11]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12_1:%.*]] = sub i32 [[CONV9_1]], [[CONV11_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12_3:%.*]] = sub i32 [[CONV9_3]], [[CONV11_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12_2:%.*]] = sub i32 [[CONV9_2]], [[CONV11_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24:%.*]] = sub i32 [[CONV21]], [[CONV23]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24_1:%.*]] = sub i32 [[CONV21_1]], [[CONV23_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24_3:%.*]] = sub i32 [[CONV21_3]], [[CONV23_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24_2:%.*]] = sub i32 [[CONV21_2]], [[CONV23_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36:%.*]] = sub i32 [[CONV33]], [[CONV35]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36_1:%.*]] = sub i32 [[CONV33_1]], [[CONV35_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36_3:%.*]] = sub i32 [[CONV33_3]], [[CONV35_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36_2:%.*]] = sub i32 [[CONV33_2]], [[CONV35_2]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP102:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV4:%.*]] = zext i8 [[TMP5]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV4_1:%.*]] = zext i8 [[TMP21]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP61]] to i32
; UNALIGNED_VEC_MEM-NEXT: [[CONV40_2:%.*]] = zext i8 [[TMP42]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV4_2:%.*]] = zext i8 [[TMP41]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14:%.*]] = zext i8 [[TMP4]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14_1:%.*]] = zext i8 [[TMP20]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14_3:%.*]] = zext i8 [[TMP57]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14_2:%.*]] = zext i8 [[TMP40]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26:%.*]] = zext i8 [[TMP3]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26_1:%.*]] = zext i8 [[TMP19]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26_3:%.*]] = zext i8 [[TMP61]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26_2:%.*]] = zext i8 [[TMP39]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38:%.*]] = zext i8 [[TMP51]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_1:%.*]] = zext i8 [[TMP18]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP102]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_2:%.*]] = zext i8 [[TMP38]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP62:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP63:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP64:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP65:%.*]] = load i8, ptr [[ARRAYIDX5_4]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6:%.*]] = zext i8 [[TMP29]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6_1:%.*]] = zext i8 [[TMP25]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6_3:%.*]] = zext i8 [[TMP65]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6_2:%.*]] = zext i8 [[TMP45]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16:%.*]] = zext i8 [[TMP28]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16_1:%.*]] = zext i8 [[TMP24]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16_3:%.*]] = zext i8 [[TMP64]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16_2:%.*]] = zext i8 [[TMP44]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28:%.*]] = zext i8 [[TMP27]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28_1:%.*]] = zext i8 [[TMP23]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28_3:%.*]] = zext i8 [[TMP63]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28_2:%.*]] = zext i8 [[TMP43]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40:%.*]] = zext i8 [[TMP26]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40_1:%.*]] = zext i8 [[TMP22]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40_3:%.*]] = zext i8 [[TMP62]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40_4:%.*]] = zext i8 [[TMP78]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7:%.*]] = sub i32 [[CONV4]], [[CONV6]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7_1:%.*]] = sub i32 [[CONV4_1]], [[CONV6_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7_3:%.*]] = sub i32 [[CONV40_2]], [[CONV6_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7_2:%.*]] = sub i32 [[CONV4_2]], [[CONV6_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17:%.*]] = sub i32 [[CONV14]], [[CONV16]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17_1:%.*]] = sub i32 [[CONV14_1]], [[CONV16_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17_3:%.*]] = sub i32 [[CONV14_3]], [[CONV16_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17_2:%.*]] = sub i32 [[CONV14_2]], [[CONV16_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29:%.*]] = sub i32 [[CONV26]], [[CONV28]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29_1:%.*]] = sub i32 [[CONV26_1]], [[CONV28_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29_3:%.*]] = sub i32 [[CONV26_3]], [[CONV28_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29_2:%.*]] = sub i32 [[CONV26_2]], [[CONV28_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41:%.*]] = sub i32 [[CONV38]], [[CONV40]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41_1:%.*]] = sub i32 [[CONV38_1]], [[CONV40_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41_3:%.*]] = sub i32 [[CONV38_3]], [[CONV40_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41_2:%.*]] = sub i32 [[CONV38_2]], [[CONV40_4]]
-; UNALIGNED_VEC_MEM-NEXT: [[SHL:%.*]] = shl i32 [[SUB7]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL_1:%.*]] = shl i32 [[SUB7_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL_3:%.*]] = shl i32 [[SUB7_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL_2:%.*]] = shl i32 [[SUB7_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18:%.*]] = shl i32 [[SUB17]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18_1:%.*]] = shl i32 [[SUB17_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18_3:%.*]] = shl i32 [[SUB17_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18_2:%.*]] = shl i32 [[SUB17_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30:%.*]] = shl i32 [[SUB29]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30_1:%.*]] = shl i32 [[SUB29_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30_3:%.*]] = shl i32 [[SUB29_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30_2:%.*]] = shl i32 [[SUB29_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42:%.*]] = shl i32 [[SUB41]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42_1:%.*]] = shl i32 [[SUB41_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42_3:%.*]] = shl i32 [[SUB41_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42_2:%.*]] = shl i32 [[SUB41_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = add i32 [[SHL]], [[SUB]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = add i32 [[SHL_1]], [[SUB_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = add i32 [[SHL_3]], [[SUB_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = add i32 [[SHL_2]], [[SUB_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = add i32 [[SHL18]], [[SUB12]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = add i32 [[SHL18_1]], [[SUB12_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = add i32 [[SHL18_3]], [[SUB12_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = add i32 [[SHL18_2]], [[SUB12_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = add i32 [[SHL30]], [[SUB24]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = add i32 [[SHL30_1]], [[SUB24_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = add i32 [[SHL30_3]], [[SUB24_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = add i32 [[SHL30_2]], [[SUB24_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = add i32 [[SHL42]], [[SUB36]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = add i32 [[SHL42_1]], [[SUB36_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = add i32 [[SHL42_3]], [[SUB36_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = add i32 [[SHL42_2]], [[SUB36_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = insertelement <2 x i8> poison, i8 [[TMP106]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = insertelement <2 x i8> [[TMP33]], i8 [[TMP57]], i32 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_4]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = zext <4 x i8> [[TMP35]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> poison, i32 [[CONV40_2]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = zext <2 x i8> [[TMP34]] to <2 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = shufflevector <2 x i32> [[TMP38]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = shufflevector <4 x i32> [[TMP37]], <4 x i32> [[TMP39]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[CONV38_3]], i32 3
+; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = sub <4 x i32> [[TMP41]], [[TMP36]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = shl <4 x i32> [[TMP58]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = add <4 x i32> [[TMP43]], [[TMP30]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load <4 x i8>, ptr [[ADD_PTR_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = zext <4 x i8> [[TMP45]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = zext <4 x i8> [[TMP46]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = sub <4 x i32> [[TMP49]], [[TMP50]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = zext <4 x i8> [[TMP47]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = zext <4 x i8> [[TMP48]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = sub <4 x i32> [[TMP52]], [[TMP53]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = shl <4 x i32> [[TMP54]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = add <4 x i32> [[TMP55]], [[TMP51]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45:%.*]] = sub i32 [[ADD_3]], [[ADD31_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_1:%.*]] = sub i32 [[ADD_2]], [[ADD31_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_3:%.*]] = sub i32 [[ADD19_3]], [[ADD43_4]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_2:%.*]] = sub i32 [[ADD19_2]], [[ADD43_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44:%.*]] = add i32 [[ADD31_3]], [[ADD_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_1:%.*]] = add i32 [[ADD31_2]], [[ADD_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_3:%.*]] = add i32 [[ADD43_4]], [[ADD19_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_2:%.*]] = add i32 [[ADD43_2]], [[ADD19_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47:%.*]] = sub i32 [[ADD31]], [[ADD43]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_1:%.*]] = sub i32 [[ADD31_1]], [[ADD43_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_3:%.*]] = sub i32 [[ADD31_4]], [[ADD43_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_2:%.*]] = sub i32 [[ADD31_5]], [[ADD43_5]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46:%.*]] = add i32 [[ADD43]], [[ADD31]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46_1:%.*]] = add i32 [[ADD43_1]], [[ADD31_1]]
@@ -590,18 +427,17 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: [[ADD103:%.*]] = add i32 [[ADD94]], [[ADD78]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD105:%.*]] = add i32 [[SUB102]], [[SUB86]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP66:%.*]] = insertelement <16 x i32> poison, i32 [[ADD46_2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = insertelement <16 x i32> [[TMP66]], i32 [[CONV_3]], i32 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP74]], <16 x i32> <i32 0, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP68:%.*]] = insertelement <16 x i32> [[TMP67]], i32 [[ADD46_1]], i32 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP69:%.*]] = insertelement <16 x i32> [[TMP68]], i32 [[ADD46]], i32 3
-; UNALIGNED_VEC_MEM-NEXT: [[TMP70:%.*]] = insertelement <16 x i32> [[TMP69]], i32 [[CONV_2]], i32 4
-; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = insertelement <16 x i32> [[TMP70]], i32 [[CONV9_2]], i32 5
+; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = shufflevector <4 x i32> [[TMP49]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP69]], <16 x i32> [[TMP78]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP72:%.*]] = insertelement <16 x i32> [[TMP71]], i32 [[SUB47_1]], i32 6
; UNALIGNED_VEC_MEM-NEXT: [[TMP73:%.*]] = insertelement <16 x i32> [[TMP72]], i32 [[SUB47]], i32 7
-; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = insertelement <16 x i32> [[TMP73]], i32 [[CONV_1]], i32 8
-; UNALIGNED_VEC_MEM-NEXT: [[TMP75:%.*]] = insertelement <16 x i32> [[TMP74]], i32 [[CONV9_1]], i32 9
-; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = insertelement <16 x i32> [[TMP75]], i32 [[CONV21_1]], i32 10
+; UNALIGNED_VEC_MEM-NEXT: [[TMP118:%.*]] = shufflevector <4 x i32> [[TMP18]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = shufflevector <16 x i32> [[TMP73]], <16 x i32> [[TMP118]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP77:%.*]] = insertelement <16 x i32> [[TMP76]], i32 [[ADD44]], i32 11
-; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
; UNALIGNED_VEC_MEM-NEXT: [[TMP79:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP80:%.*]] = shufflevector <16 x i32> [[TMP77]], <16 x i32> [[TMP79]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
; UNALIGNED_VEC_MEM-NEXT: [[TMP81:%.*]] = lshr <16 x i32> [[TMP80]], splat (i32 15)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 3fb5fd9c6b5dc..698f765e3ad5c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -4,7 +4,8 @@
define <4 x i16> @test() {
; CHECK-LABEL: define <4 x i16> @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, zeroinitializer
+; CHECK-NEXT: [[SUBI:%.*]] = add <4 x i16> zeroinitializer, zeroinitializer
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, [[SUBI]]
; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
>From f07e30178f102951f4098f9a0f85ec2037612d74 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Mon, 25 May 2026 00:56:01 -0700
Subject: [PATCH 3/8] address reviewer's comments
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 28 +++++++++++--------
.../revec-reduced-value-vectorized-later.ll | 3 +-
2 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0e84153ad6e5e..39e152f912504 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3877,11 +3877,7 @@ class slpvectorizer::BoUpSLP {
AnalyzedReductionVals.insert(hash_value(VL));
}
/// Clear the list of the analyzed reduction root instructions.
- void clearReductionData() {
- AnalyzedReductionsRoots.clear();
- AnalyzedReductionVals.clear();
- AnalyzedMinBWVals.clear();
- }
+ void clearReductionData();
/// Checks if the given value is gathered in one of the nodes.
bool isAnyGathered(const SmallDenseSet<Value *> &Vals) const {
return any_of(MustGather, [&](Value *V) { return Vals.contains(V); });
@@ -28494,6 +28490,8 @@ class HorizontalReduction {
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
+ /// List of values to ignore while collecting reduction values.
+ inline static SmallDenseSet<Value *> IgnoredVals;
/// Maps reduced value to the corresponding reduction operation.
SmallDenseMap<Value *, SmallVector<Instruction *>, 16> ReducedValsToOps;
WeakTrackingVH ReductionRoot;
@@ -29062,6 +29060,8 @@ class HorizontalReduction {
Operands.contains(EdgeInst) ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>))) {
+ if (IgnoredVals.contains(EdgeVal))
+ continue;
PossibleReducedVals.push_back(EdgeVal);
if (EdgeInst && !isCmpSelMinMax(EdgeInst))
Operands.insert_range(EdgeInst->operands());
@@ -29069,7 +29069,8 @@ class HorizontalReduction {
}
if (CurrentRK == ReductionOrdering::Ordered)
RK = ReductionOrdering::Ordered;
- ReductionOps.push_back(EdgeInst);
+ if (!IgnoredVals.contains(EdgeVal))
+ ReductionOps.push_back(EdgeInst);
}
};
// Try to regroup reduced values so that it gets more profitable to try to
@@ -30280,11 +30281,13 @@ class HorizontalReduction {
auto Position = I * DestTyNumElements;
Value *SubVec =
createExtractVector(Builder, Vec, DestTyNumElements, Position);
+ IgnoredVals.insert(SubVec);
if (!Rdx) {
Rdx = SubVec;
} else {
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
}
+ IgnoredVals.insert(Rdx);
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
@@ -30384,8 +30387,6 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
- auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
- ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
@@ -30896,6 +30897,14 @@ class HorizontalReduction {
static RecurKind getRdxKind(Value *V) {
return HorizontalReduction::getRdxKind(V);
}
+
+void BoUpSLP::clearReductionData() {
+ AnalyzedReductionsRoots.clear();
+ AnalyzedReductionVals.clear();
+ AnalyzedMinBWVals.clear();
+ HorizontalReduction::clearReductionData();
+}
+
static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) {
if (auto *IE = dyn_cast<InsertElementInst>(InsertInst))
return cast<FixedVectorType>(IE->getType())->getNumElements();
@@ -31190,9 +31199,6 @@ bool SLPVectorizerPass::vectorizeHorReduction(
Res = true;
if (auto *I = dyn_cast<Instruction>(VectorizedV); I && I != Inst) {
// Try to find another reduction.
- if (SLPReVec && I->getType()->isVectorTy() &&
- Inst->getType()->isVectorTy())
- continue;
Stack.emplace(I, Level);
continue;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 698f765e3ad5c..3fb5fd9c6b5dc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -4,8 +4,7 @@
define <4 x i16> @test() {
; CHECK-LABEL: define <4 x i16> @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[SUBI:%.*]] = add <4 x i16> zeroinitializer, zeroinitializer
-; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, [[SUBI]]
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, zeroinitializer
; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
>From 1802f1af9ca5c2aad03dd062bb0e48e53a82d7df Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 2 Jun 2026 04:07:10 -0700
Subject: [PATCH 4/8] address reviewer's comments
Reject reduction tree with 3 nodes.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 40 +++++++++++--------
llvm/test/Transforms/SLPVectorizer/revec.ll | 29 ++++++++++++++
2 files changed, 52 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 39e152f912504..7ff3ec8568079 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3877,7 +3877,11 @@ class slpvectorizer::BoUpSLP {
AnalyzedReductionVals.insert(hash_value(VL));
}
/// Clear the list of the analyzed reduction root instructions.
- void clearReductionData();
+ void clearReductionData() {
+ AnalyzedReductionsRoots.clear();
+ AnalyzedReductionVals.clear();
+ AnalyzedMinBWVals.clear();
+ }
/// Checks if the given value is gathered in one of the nodes.
bool isAnyGathered(const SmallDenseSet<Value *> &Vals) const {
return any_of(MustGather, [&](Value *V) { return Vals.contains(V); });
@@ -18044,6 +18048,21 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
if (!DebugCounter::shouldExecute(VectorizedGraphs))
return true;
+ // If we are revectorizing reduction, it may result in same reduction pattern
+ // with shuffles as leaves of the reduction. Prevent SLP from revectorizing
+ // that shuffle pattern.
+ if (SLPReVec && ForReduction && VectorizableTree.size() == 3 &&
+ VectorizableTree[0]->State == TreeEntry::Vectorize &&
+ VectorizableTree[0]->getOpcode() == Instruction::ShuffleVector &&
+ VectorizableTree[1]->isGather() &&
+ isSplat(VectorizableTree[1]->Scalars) &&
+ VectorizableTree[2]->isGather() &&
+ allConstant(VectorizableTree[2]->Scalars)) {
+ LLVM_DEBUG(dbgs() << "SLP: Rejecting reduction tree with 3 nodes(shuffle "
+ "as root and remaining are gather nodes).\n");
+ return true;
+ }
+
// Graph is empty - do nothing.
if (VectorizableTree.empty()) {
assert(ExternalUses.empty() && "We shouldn't have any external users");
@@ -28490,8 +28509,6 @@ class HorizontalReduction {
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
- /// List of values to ignore while collecting reduction values.
- inline static SmallDenseSet<Value *> IgnoredVals;
/// Maps reduced value to the corresponding reduction operation.
SmallDenseMap<Value *, SmallVector<Instruction *>, 16> ReducedValsToOps;
WeakTrackingVH ReductionRoot;
@@ -29060,8 +29077,6 @@ class HorizontalReduction {
Operands.contains(EdgeInst) ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>))) {
- if (IgnoredVals.contains(EdgeVal))
- continue;
PossibleReducedVals.push_back(EdgeVal);
if (EdgeInst && !isCmpSelMinMax(EdgeInst))
Operands.insert_range(EdgeInst->operands());
@@ -29069,8 +29084,7 @@ class HorizontalReduction {
}
if (CurrentRK == ReductionOrdering::Ordered)
RK = ReductionOrdering::Ordered;
- if (!IgnoredVals.contains(EdgeVal))
- ReductionOps.push_back(EdgeInst);
+ ReductionOps.push_back(EdgeInst);
}
};
// Try to regroup reduced values so that it gets more profitable to try to
@@ -30281,13 +30295,11 @@ class HorizontalReduction {
auto Position = I * DestTyNumElements;
Value *SubVec =
createExtractVector(Builder, Vec, DestTyNumElements, Position);
- IgnoredVals.insert(SubVec);
if (!Rdx) {
Rdx = SubVec;
} else {
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
}
- IgnoredVals.insert(Rdx);
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
@@ -30387,6 +30399,8 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
+ auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
+ ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
@@ -30897,14 +30911,6 @@ class HorizontalReduction {
static RecurKind getRdxKind(Value *V) {
return HorizontalReduction::getRdxKind(V);
}
-
-void BoUpSLP::clearReductionData() {
- AnalyzedReductionsRoots.clear();
- AnalyzedReductionVals.clear();
- AnalyzedMinBWVals.clear();
- HorizontalReduction::clearReductionData();
-}
-
static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) {
if (auto *IE = dyn_cast<InsertElementInst>(InsertInst))
return cast<FixedVectorType>(IE->getType())->getNumElements();
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 2849c82142b55..8a813d94af68b 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -469,3 +469,32 @@ entry:
store <4 x float> %11, ptr %1, align 16
ret i32 0
}
+
+define <4 x i32> @hor_reduction_four_points(ptr %a) {
+; CHECK-LABEL: @hor_reduction_four_points(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = add <4 x i32> [[RDX_OP]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[RDX_OP2:%.*]] = add <4 x i32> [[RDX_OP1]], [[TMP4]]
+; CHECK-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP2]], zeroinitializer
+; CHECK-NEXT: ret <4 x i32> [[OP_RDX]]
+;
+entry:
+ %gep1 = getelementptr <4 x i32>, ptr %a, i64 1
+ %gep2 = getelementptr <4 x i32>, ptr %a, i64 2
+ %gep3 = getelementptr <4 x i32>, ptr %a, i64 3
+ %a0 = load <4 x i32>, ptr %a
+ %a1 = load <4 x i32>, ptr %gep1
+ %a2 = load <4 x i32>, ptr %gep2
+ %a3 = load <4 x i32>, ptr %gep3
+ %add0 = add <4 x i32> zeroinitializer, %a0
+ %add1 = add <4 x i32> %add0, %a1
+ %add2 = add <4 x i32> %add1, %a2
+ %add3 = add <4 x i32> %add2, %a3
+ ret <4 x i32> %add3
+}
>From da426774f2614bab72f7bb802fb3ec5ec4fcfc47 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Wed, 3 Jun 2026 08:47:48 -0700
Subject: [PATCH 5/8] address unused variable error
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7ff3ec8568079..4f4a05262605f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30399,8 +30399,6 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
- auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
- ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
>From b1b377ac1afee98509ba41a78334302118851d3b Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 4 Jun 2026 01:33:51 -0700
Subject: [PATCH 6/8] address formatting issues
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 37 ++++++++++---------
1 file changed, 20 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4f4a05262605f..7d83f8e591b53 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30276,30 +30276,33 @@ class HorizontalReduction {
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
+ // e.g. Consider vector reduce add.
+ // Initial reduction is
+ // clang-format off
+ // %add0 = add <4 x i32> zeroinitializer, %A
+ // %add1 = add <4 x i32> %add0, %B
+ //
+ // where,
+ // %A = <a, b, c, d>
+ // %B = <e, f, g, h>
+ // %add1 = A + B = <a + e, b + f, c + g, d + h>
+ //
+ // After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
+ // the reduction can be expressed as:
+ // %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ // %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ // %add0 = add <4 x i32> zeroinitializer, %A
+ // %add1 = add <4 x i32> %add0, %B
+ // clang-format on
Rdx = nullptr;
- /*
- e.g. Consider vector reduce add.
-
- RdxVal[0] = [a, b, c, d]
- RdxVal[1] = [e, f, g, h]
- Add0 = zeroinitializer + RdxVal[0]
- Add1 = Add0 + RdxVal[1]
-
- After revectorization with VF=2 and Vec = [a, b, c, d, e, f, g, h],
- the reduction can be expressed as:
- RdxVal[0] = ExtractVector(Vec, 0, 3) = [a, b, c, d]
- RdxVal[1] = ExtractVector(Vec, 4, 7) = [e, f, g, h]
- Add = RdxVal[0] + RdxVal[1]
- */
for (auto I : seq<unsigned>(VF)) {
auto Position = I * DestTyNumElements;
Value *SubVec =
createExtractVector(Builder, Vec, DestTyNumElements, Position);
- if (!Rdx) {
+ if (!Rdx)
Rdx = SubVec;
- } else {
+ else
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
- }
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
>From b8e2ccf9bd7dae340e5d794596c9a975836d0d69 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 4 Jun 2026 19:26:20 -0700
Subject: [PATCH 7/8] consider FMF in arithmetic instruction cost and other
minor issues addressed
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 14 +++++++++++---
llvm/test/Transforms/SLPVectorizer/revec.ll | 4 ++--
2 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7d83f8e591b53..983294cfd167d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30402,12 +30402,20 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
- for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
+ for (unsigned I : seq<unsigned>(ReducedVals.size())) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
CostKind, I * ScalarTyNumElements, VecTy);
- VectorCost +=
- TTI->getArithmeticInstrCost(RdxOpcode, VecTy, CostKind);
+ }
+ // We get one less arithmetic instruction compared to number of
+ // reduced values. We are also passing CtxI so that the backend can
+ // synthesize FMF from the respective reduction instruction.
+ for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
+ VectorCost += TTI->getArithmeticInstrCost(
+ RdxOpcode, VecTy, CostKind,
+ TTI::getOperandInfo(ReducedVals[I]),
+ TTI::getOperandInfo(ReducedVals[I + 1]), {},
+ ReducedValsToOps[ReducedVals[I]].front());
}
} else {
Type *RedTy = VectorTy->getElementType();
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 8a813d94af68b..346669c53824f 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -481,7 +481,7 @@ define <4 x i32> @hor_reduction_four_points(ptr %a) {
; CHECK-NEXT: [[RDX_OP1:%.*]] = add <4 x i32> [[RDX_OP]], [[TMP3]]
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[RDX_OP2:%.*]] = add <4 x i32> [[RDX_OP1]], [[TMP4]]
-; CHECK-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP2]], zeroinitializer
+; CHECK-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP2]], <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <4 x i32> [[OP_RDX]]
;
entry:
@@ -492,7 +492,7 @@ entry:
%a1 = load <4 x i32>, ptr %gep1
%a2 = load <4 x i32>, ptr %gep2
%a3 = load <4 x i32>, ptr %gep3
- %add0 = add <4 x i32> zeroinitializer, %a0
+ %add0 = add <4 x i32> <i32 0, i32 1, i32 2, i32 3>, %a0
%add1 = add <4 x i32> %add0, %a1
%add2 = add <4 x i32> %add1, %a2
%add3 = add <4 x i32> %add2, %a3
>From a648656347e57c9c7e2100ee22a57d9fbfc63d9a Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 9 Jun 2026 17:47:51 -0700
Subject: [PATCH 8/8] Make scalar cost computation better by passing the
reduction instruction to scalar cost compute function
Passing reduction instruction should help getArithmeticInstrCost in synthesizing FMF from the instruction
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 105 ++++++++++--------
1 file changed, 56 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 983294cfd167d..d02f051c9c532 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30331,57 +30331,61 @@ class HorizontalReduction {
// If all of the reduced values are constant, the vector cost is 0, since
// the reduction value can be calculated at the compile time.
bool AllConsts = allConstant(ReducedVals);
- auto EvaluateScalarCost = [&](function_ref<InstructionCost()> GenCostFn) {
- InstructionCost Cost = 0;
- // Scalar cost is repeated for N-1 elements.
- int Cnt = ReducedVals.size();
- for (Value *RdxVal : ReducedVals) {
- if (!isa<Instruction>(RdxVal))
- continue;
- if (Cnt == 1) {
- unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
- Cost += (SameValueCount ? SameValueCount - 1 : 0) * GenCostFn();
- break;
- }
- --Cnt;
- if (RdxVal->hasNUsesOrMore(IsCmpSelMinMax ? 3 : 2)) {
- unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
- Cost += (SameValueCount ? SameValueCount : 1) * GenCostFn();
- continue;
- }
- InstructionCost ScalarCost = 0;
- for (User *U : RdxVal->users()) {
- auto *RdxOp = cast<Instruction>(U);
- if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
- if (RdxKind == RecurKind::FAdd) {
- InstructionCost FMACost = canConvertToFMA(
- RdxOp, getSameOpcode(RdxOp, TLI), DT, DL, *TTI, TLI);
- if (FMACost.isValid()) {
- LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
- if (auto *I = dyn_cast<Instruction>(RdxVal)) {
- // Also, exclude scalar fmul cost.
- InstructionCost FMulCost =
- TTI->getInstructionCost(I, CostKind);
- LLVM_DEBUG(dbgs() << "Minus FMul cost: " << FMulCost << "\n");
- FMACost -= FMulCost;
+ auto EvaluateScalarCost =
+ [&](function_ref<InstructionCost(Instruction *)> GenCostFn) {
+ InstructionCost Cost = 0;
+ // Scalar cost is repeated for N-1 elements.
+ int Cnt = ReducedVals.size();
+ for (auto [Idx, RdxVal] : enumerate(ReducedVals)) {
+ if (!isa<Instruction>(RdxVal))
+ continue;
+ Instruction *RdxOp = ReducedValsToOps[ReducedVals[Idx]].front();
+ if (Cnt == 1) {
+ unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
+ Cost +=
+ (SameValueCount ? SameValueCount - 1 : 0) * GenCostFn(RdxOp);
+ break;
+ }
+ --Cnt;
+ if (RdxVal->hasNUsesOrMore(IsCmpSelMinMax ? 3 : 2)) {
+ unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
+ Cost += (SameValueCount ? SameValueCount : 1) * GenCostFn(RdxOp);
+ continue;
+ }
+ InstructionCost ScalarCost = 0;
+ for (User *U : RdxVal->users()) {
+ auto *RdxOp = cast<Instruction>(U);
+ if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
+ if (RdxKind == RecurKind::FAdd) {
+ InstructionCost FMACost = canConvertToFMA(
+ RdxOp, getSameOpcode(RdxOp, TLI), DT, DL, *TTI, TLI);
+ if (FMACost.isValid()) {
+ LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
+ if (auto *I = dyn_cast<Instruction>(RdxVal)) {
+ // Also, exclude scalar fmul cost.
+ InstructionCost FMulCost =
+ TTI->getInstructionCost(I, CostKind);
+ LLVM_DEBUG(dbgs()
+ << "Minus FMul cost: " << FMulCost << "\n");
+ FMACost -= FMulCost;
+ }
+ ScalarCost += FMACost;
+ continue;
+ }
}
- ScalarCost += FMACost;
+ ScalarCost += TTI->getInstructionCost(RdxOp, CostKind);
continue;
}
+ ScalarCost = InstructionCost::getInvalid();
+ break;
}
- ScalarCost += TTI->getInstructionCost(RdxOp, CostKind);
- continue;
+ if (ScalarCost.isValid())
+ Cost += ScalarCost;
+ else
+ Cost += GenCostFn(RdxOp);
}
- ScalarCost = InstructionCost::getInvalid();
- break;
- }
- if (ScalarCost.isValid())
- Cost += ScalarCost;
- else
- Cost += GenCostFn();
- }
- return Cost;
- };
+ return Cost;
+ };
// Require reduction cost if:
// 1. This type is not a full register type and no other vectors with the
// same type in the storage (first vector with small type).
@@ -30485,8 +30489,11 @@ class HorizontalReduction {
}
}
}
- ScalarCost = EvaluateScalarCost([&]() {
- return TTI->getArithmeticInstrCost(RdxOpcode, ScalarTy, CostKind);
+ ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
+ return TTI->getArithmeticInstrCost(
+ RdxOpcode, ScalarTy, CostKind,
+ TTI::getOperandInfo(RdxOp->getOperand(0)),
+ TTI::getOperandInfo(RdxOp->getOperand(1)), {}, RdxOp);
});
break;
}
@@ -30521,7 +30528,7 @@ class HorizontalReduction {
}
}
}
- ScalarCost = EvaluateScalarCost([&]() {
+ ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
IntrinsicCostAttributes ICA(Id, ScalarTy, {ScalarTy, ScalarTy}, FMF);
return TTI->getIntrinsicInstrCost(ICA, CostKind);
});
More information about the llvm-commits
mailing list