[llvm] [SLP] Inefficient cost-modelling and codegen for reductions with slp-… (PR #197875)
Sushant Gokhale via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 18:56:32 PDT 2026
https://github.com/sushgokh updated https://github.com/llvm/llvm-project/pull/197875
>From b0add207488f1fd093b5a92a99c5097f3514cc85 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 14 May 2026 23:35:29 -0700
Subject: [PATCH 1/9] [SLP] Inefficient cost-modelling and codegen for
reductions with slp-revec
When revectorizing, starting with reduction, SLP generates slightly inefficient code for reduction.
e.g. In the godbolt link https://godbolt.org/z/ez7KPnxM5,
hor_reduction --> original code
hor_reduction_revec_as_imagined_in_SLP --> revectorized code would look like
Rather than extracting per lane, we can extract original leaf nodes of the reduction, which are sub-vectors, and then perform usual reduction as in non-revectorized code. In the above link,
hor_reduction_ideal_revec --> how the revec code should look like
Extracting subvectors and achieving the reduction result would be better than extracting per lane and achieving the same result.
Change in Transforms/SLPVectorizer/RISCV/complex-loads.ll is side effect of this change. The change with the file seems to better. IR post-SLP before/after patch and llvm-mca analysis of the corresponding codegen can be found below:
IR post-SLP before patch: https://godbolt.org/z/q1b7nW416
IR post-SLP after patch: https://godbolt.org/z/W6vsP4zox
codegen comparison: https://godbolt.org/z/TGzjn3K8T
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 86 ++---
.../SLPVectorizer/RISCV/complex-loads.ll | 314 +++++++++++++-----
.../SLPVectorizer/SystemZ/revec-fix-128169.ll | 19 +-
.../revec-reduced-value-vectorized-later.ll | 38 +--
llvm/test/Transforms/SLPVectorizer/revec.ll | 19 +-
5 files changed, 304 insertions(+), 172 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5210088ded294..f7675b8683bd0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29010,6 +29010,7 @@ class HorizontalReduction {
// Also, do not try to reduce const values, if the operation is not
// foldable.
bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
+ R.isAnalyzedReductionRoot(EdgeInst) ||
getRdxKind(EdgeInst) != RdxKind ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
ReductionOrdering CurrentRK = IsReducedVal
@@ -29826,10 +29827,11 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered)
return VectorizedTree;
- if (!VectorValuesAndScales.empty())
+ if (!VectorValuesAndScales.empty()) {
VectorizedTree = GetNewVectorizedTree(
VectorizedTree,
- emitReduction(Builder, *TTI, ReductionRoot->getType()));
+ emitReduction(V, Builder, *TTI, ReductionRoot->getType()));
+ }
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@@ -30221,8 +30223,9 @@ class HorizontalReduction {
private:
/// Creates the reduction from the given \p Vec vector value with the given
/// scale \p Scale and signedness \p IsSigned.
- Value *createSingleOp(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Value *Vec, unsigned Scale, bool IsSigned, Type *DestTy,
+ Value *createSingleOp(BoUpSLP &V, IRBuilderBase &Builder,
+ const TargetTransformInfo &TTI, Value *Vec,
+ unsigned Scale, bool IsSigned, Type *DestTy,
bool ReducedInTree) {
Value *Rdx;
if (ReducedInTree) {
@@ -30230,25 +30233,38 @@ class HorizontalReduction {
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
- Rdx = PoisonValue::get(
- getWidenedType(Vec->getType()->getScalarType(), DestTyNumElements));
- for (unsigned I : seq<unsigned>(DestTyNumElements)) {
- // Do reduction for each lane.
- // e.g., do reduce add for
- // VL[0] = <4 x Ty> <a, b, c, d>
- // VL[1] = <4 x Ty> <e, f, g, h>
- // Lane[0] = <2 x Ty> <a, e>
- // Lane[1] = <2 x Ty> <b, f>
- // Lane[2] = <2 x Ty> <c, g>
- // Lane[3] = <2 x Ty> <d, h>
- // result[0] = reduce add Lane[0]
- // result[1] = reduce add Lane[1]
- // result[2] = reduce add Lane[2]
- // result[3] = reduce add Lane[3]
- SmallVector<int, 16> Mask = createStrideMask(I, DestTyNumElements, VF);
- Value *Lane = Builder.CreateShuffleVector(Vec, Mask);
- Rdx = Builder.CreateInsertElement(
- Rdx, emitReduction(Lane, Builder, &TTI, DestTy), I);
+ unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
+ Rdx = nullptr;
+ /*
+ e.g. Consider vector reduce add.
+
+ Initial reduction is
+ %add0 = add <4 x i32> zeroinitializer, %A
+ %add1 = add <4 x i32> %add0, %B
+
+ where,
+ %A = <a, b, c, d>
+ %B = <e, f, g, h>
+ %add1 = A + B = <a + e, b + f, c + g, d + h>
+
+ After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
+ the reduction can be expressed as:
+ %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %add0 = add <4 x i32> zeroinitializer, %A
+ %add1 = add <4 x i32> %add0, %B
+ */
+ for (auto I : seq<unsigned>(VF)) {
+ auto Position = I * DestTyNumElements;
+ Value *SubVec =
+ createExtractVector(Builder, Vec, DestTyNumElements, Position);
+ if (!Rdx) {
+ Rdx = SubVec;
+ } else {
+ Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
+ if (isa<Instruction>(Rdx))
+ V.analyzedReductionRoot(cast<Instruction>(Rdx));
+ }
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
@@ -30348,19 +30364,15 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
+ auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
+ ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
- VectorCost += TTI->getShuffleCost(
- TTI::SK_PermuteSingleSrc,
- FixedVectorType::get(VecTy->getScalarType(),
- ReducedVals.size()),
- VectorTy,
- createStrideMask(I, ScalarTyNumElements, ReducedVals.size()));
- VectorCost += TTI->getArithmeticReductionCost(RdxOpcode, VecTy,
- FMF, CostKind);
+ VectorCost += TTI->getShuffleCost(TTI::SK_ExtractSubvector, VecTy,
+ DstTy, {}, CostKind,
+ I * ScalarTyNumElements, VecTy);
+ VectorCost +=
+ TTI->getArithmeticInstrCost(RdxOpcode, VecTy, CostKind);
}
- VectorCost += TTI->getScalarizationOverhead(
- VecTy, APInt::getAllOnes(ScalarTyNumElements), /*Insert*/ true,
- /*Extract*/ false, TTI::TCK_RecipThroughput);
} else {
Type *RedTy = VectorTy->getElementType();
auto [RType, IsSigned] = R.getRootNodeTypeWithNoCast().value_or(
@@ -30484,13 +30496,13 @@ class HorizontalReduction {
/// Splits the values, stored in VectorValuesAndScales, into registers/free
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
- Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Type *DestTy) {
+ Value *emitReduction(BoUpSLP &V, IRBuilderBase &Builder,
+ const TargetTransformInfo &TTI, Type *DestTy) {
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
bool ReducedInTree) {
- Value *Rdx = createSingleOp(Builder, TTI, Vec, Scale, IsSigned, DestTy,
+ Value *Rdx = createSingleOp(V, Builder, TTI, Vec, Scale, IsSigned, DestTy,
ReducedInTree);
if (ReducedSubTree)
ReducedSubTree = createOp(Builder, RdxKind, ReducedSubTree, Rdx,
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index 09be099fcf709..ed3f85d31d314 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -277,102 +277,265 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: entry:
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20:%.*]] = getelementptr i8, ptr [[PIX1]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22:%.*]] = getelementptr i8, ptr [[PIX2]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25:%.*]] = getelementptr i8, ptr [[PIX1]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27:%.*]] = getelementptr i8, ptr [[PIX2]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34:%.*]] = getelementptr i8, ptr [[PIX2]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37:%.*]] = getelementptr i8, ptr [[PIX1]], i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39:%.*]] = getelementptr i8, ptr [[PIX2]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR644:%.*]] = getelementptr i8, ptr [[PIX2]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR_1:%.*]] = getelementptr i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_4:%.*]] = getelementptr i8, ptr null, i64 4
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_3:%.*]] = getelementptr i8, ptr null, i64 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_3:%.*]] = getelementptr i8, ptr null, i64 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP57:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_3:%.*]] = getelementptr i8, ptr null, i64 5
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_3:%.*]] = getelementptr i8, ptr null, i64 2
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_3:%.*]] = getelementptr i8, ptr null, i64 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP61:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_3:%.*]] = getelementptr i8, ptr null, i64 6
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_3:%.*]] = getelementptr i8, ptr null, i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_3:%.*]] = getelementptr i8, ptr null, i64 3
+; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_3:%.*]] = getelementptr i8, ptr null, i64 7
+; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX37]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX25]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX13]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP60:%.*]] = load i8, ptr [[ARRAYIDX32_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX20_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX8_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = load i8, ptr [[ADD_PTR3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX34_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX22_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX10_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = load i8, ptr [[ADD_PTR644]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX34]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX22]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load i8, ptr [[ARRAYIDX10]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load i8, ptr [[PIX2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX37_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = load i8, ptr [[ARRAYIDX25_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = load i8, ptr [[ARRAYIDX13_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX3_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX39_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX15_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX5_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX39]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX15]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP32:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV_1:%.*]] = zext i8 [[TMP9]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV_3:%.*]] = zext i8 [[TMP49]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV_2:%.*]] = zext i8 [[TMP33]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9_1:%.*]] = zext i8 [[TMP8]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9_3:%.*]] = zext i8 [[TMP48]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9_2:%.*]] = zext i8 [[TMP32]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21_1:%.*]] = zext i8 [[TMP7]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21_3:%.*]] = zext i8 [[TMP47]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21_2:%.*]] = zext i8 [[TMP31]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[PIX1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[PIX2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[TMP3]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = sub <4 x i32> [[TMP6]], [[TMP7]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = sub <4 x i32> [[TMP9]], [[TMP10]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = shl <4 x i32> [[TMP11]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP8]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load <4 x i8>, ptr [[ADD_PTR3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load <4 x i8>, ptr [[ADD_PTR644]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = zext <4 x i8> [[TMP14]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = zext <4 x i8> [[TMP15]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = sub <4 x i32> [[TMP18]], [[TMP19]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = zext <4 x i8> [[TMP16]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = zext <4 x i8> [[TMP17]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = sub <4 x i32> [[TMP21]], [[TMP22]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = shl <4 x i32> [[TMP23]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = add <4 x i32> [[TMP24]], [[TMP20]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load <4 x i8>, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = zext <4 x i8> [[TMP26]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load <4 x i8>, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = zext <4 x i8> [[TMP28]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = sub <4 x i32> [[TMP27]], [[TMP29]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = load i8, ptr [[ARRAYIDX20]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = load i8, ptr [[ARRAYIDX8]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33:%.*]] = zext i8 [[TMP50]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV21:%.*]] = zext i8 [[TMP52]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV9:%.*]] = zext i8 [[TMP53]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33_1:%.*]] = zext i8 [[TMP60]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33_3:%.*]] = zext i8 [[TMP46]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV33_2:%.*]] = zext i8 [[TMP30]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP106:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2:%.*]] = zext i8 [[TMP17]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2_1:%.*]] = zext i8 [[TMP13]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2_3:%.*]] = zext i8 [[TMP106]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV2_2:%.*]] = zext i8 [[TMP37]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11:%.*]] = zext i8 [[TMP16]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11_1:%.*]] = zext i8 [[TMP12]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11_3:%.*]] = zext i8 [[TMP56]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV11_2:%.*]] = zext i8 [[TMP36]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23:%.*]] = zext i8 [[TMP15]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23_1:%.*]] = zext i8 [[TMP11]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23_3:%.*]] = zext i8 [[TMP55]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV23_2:%.*]] = zext i8 [[TMP35]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35:%.*]] = zext i8 [[TMP14]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35_1:%.*]] = zext i8 [[TMP10]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35_3:%.*]] = zext i8 [[TMP54]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV35_2:%.*]] = zext i8 [[TMP34]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[TMP59:%.*]] = zext i8 [[TMP58]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[SUB:%.*]] = sub i32 [[TMP59]], [[CONV2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB_1:%.*]] = sub i32 [[CONV_1]], [[CONV2_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB_3:%.*]] = sub i32 [[CONV_3]], [[CONV2_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB_2:%.*]] = sub i32 [[CONV_2]], [[CONV2_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12:%.*]] = sub i32 [[CONV9]], [[CONV11]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12_1:%.*]] = sub i32 [[CONV9_1]], [[CONV11_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12_3:%.*]] = sub i32 [[CONV9_3]], [[CONV11_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB12_2:%.*]] = sub i32 [[CONV9_2]], [[CONV11_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24:%.*]] = sub i32 [[CONV21]], [[CONV23]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24_1:%.*]] = sub i32 [[CONV21_1]], [[CONV23_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24_3:%.*]] = sub i32 [[CONV21_3]], [[CONV23_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB24_2:%.*]] = sub i32 [[CONV21_2]], [[CONV23_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36:%.*]] = sub i32 [[CONV33]], [[CONV35]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36_1:%.*]] = sub i32 [[CONV33_1]], [[CONV35_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36_3:%.*]] = sub i32 [[CONV33_3]], [[CONV35_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB36_2:%.*]] = sub i32 [[CONV33_2]], [[CONV35_2]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP61]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP102:%.*]] = load i8, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV4:%.*]] = zext i8 [[TMP5]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV4_1:%.*]] = zext i8 [[TMP21]] to i32
; UNALIGNED_VEC_MEM-NEXT: [[CONV40_2:%.*]] = zext i8 [[TMP42]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = insertelement <2 x i8> poison, i8 [[TMP106]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = insertelement <2 x i8> [[TMP33]], i8 [[TMP57]], i32 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_4]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = zext <4 x i8> [[TMP35]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> poison, i32 [[CONV40_2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = zext <2 x i8> [[TMP34]] to <2 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = shufflevector <2 x i32> [[TMP38]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = shufflevector <4 x i32> [[TMP37]], <4 x i32> [[TMP39]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[CONV38_3]], i32 3
-; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = sub <4 x i32> [[TMP41]], [[TMP36]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = shl <4 x i32> [[TMP58]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = add <4 x i32> [[TMP43]], [[TMP30]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load <4 x i8>, ptr [[ADD_PTR_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = zext <4 x i8> [[TMP45]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = zext <4 x i8> [[TMP46]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = sub <4 x i32> [[TMP49]], [[TMP50]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = zext <4 x i8> [[TMP47]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = zext <4 x i8> [[TMP48]] to <4 x i32>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = sub <4 x i32> [[TMP52]], [[TMP53]]
-; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = shl <4 x i32> [[TMP54]], splat (i32 16)
-; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = add <4 x i32> [[TMP55]], [[TMP51]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV4_2:%.*]] = zext i8 [[TMP41]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14:%.*]] = zext i8 [[TMP4]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14_1:%.*]] = zext i8 [[TMP20]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14_3:%.*]] = zext i8 [[TMP57]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV14_2:%.*]] = zext i8 [[TMP40]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26:%.*]] = zext i8 [[TMP3]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26_1:%.*]] = zext i8 [[TMP19]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26_3:%.*]] = zext i8 [[TMP61]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV26_2:%.*]] = zext i8 [[TMP39]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38:%.*]] = zext i8 [[TMP51]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_1:%.*]] = zext i8 [[TMP18]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP102]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_2:%.*]] = zext i8 [[TMP38]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[TMP62:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP63:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP64:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP65:%.*]] = load i8, ptr [[ARRAYIDX5_4]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6:%.*]] = zext i8 [[TMP29]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6_1:%.*]] = zext i8 [[TMP25]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6_3:%.*]] = zext i8 [[TMP65]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV6_2:%.*]] = zext i8 [[TMP45]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16:%.*]] = zext i8 [[TMP28]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16_1:%.*]] = zext i8 [[TMP24]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16_3:%.*]] = zext i8 [[TMP64]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV16_2:%.*]] = zext i8 [[TMP44]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28:%.*]] = zext i8 [[TMP27]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28_1:%.*]] = zext i8 [[TMP23]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28_3:%.*]] = zext i8 [[TMP63]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV28_2:%.*]] = zext i8 [[TMP43]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40:%.*]] = zext i8 [[TMP26]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40_1:%.*]] = zext i8 [[TMP22]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40_3:%.*]] = zext i8 [[TMP62]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV40_4:%.*]] = zext i8 [[TMP78]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7:%.*]] = sub i32 [[CONV4]], [[CONV6]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7_1:%.*]] = sub i32 [[CONV4_1]], [[CONV6_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7_3:%.*]] = sub i32 [[CONV40_2]], [[CONV6_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB7_2:%.*]] = sub i32 [[CONV4_2]], [[CONV6_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17:%.*]] = sub i32 [[CONV14]], [[CONV16]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17_1:%.*]] = sub i32 [[CONV14_1]], [[CONV16_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17_3:%.*]] = sub i32 [[CONV14_3]], [[CONV16_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB17_2:%.*]] = sub i32 [[CONV14_2]], [[CONV16_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29:%.*]] = sub i32 [[CONV26]], [[CONV28]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29_1:%.*]] = sub i32 [[CONV26_1]], [[CONV28_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29_3:%.*]] = sub i32 [[CONV26_3]], [[CONV28_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB29_2:%.*]] = sub i32 [[CONV26_2]], [[CONV28_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41:%.*]] = sub i32 [[CONV38]], [[CONV40]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41_1:%.*]] = sub i32 [[CONV38_1]], [[CONV40_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41_3:%.*]] = sub i32 [[CONV38_3]], [[CONV40_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[SUB41_2:%.*]] = sub i32 [[CONV38_2]], [[CONV40_4]]
+; UNALIGNED_VEC_MEM-NEXT: [[SHL:%.*]] = shl i32 [[SUB7]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL_1:%.*]] = shl i32 [[SUB7_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL_3:%.*]] = shl i32 [[SUB7_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL_2:%.*]] = shl i32 [[SUB7_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18:%.*]] = shl i32 [[SUB17]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18_1:%.*]] = shl i32 [[SUB17_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18_3:%.*]] = shl i32 [[SUB17_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL18_2:%.*]] = shl i32 [[SUB17_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30:%.*]] = shl i32 [[SUB29]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30_1:%.*]] = shl i32 [[SUB29_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30_3:%.*]] = shl i32 [[SUB29_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL30_2:%.*]] = shl i32 [[SUB29_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42:%.*]] = shl i32 [[SUB41]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42_1:%.*]] = shl i32 [[SUB41_1]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42_3:%.*]] = shl i32 [[SUB41_3]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[SHL42_2:%.*]] = shl i32 [[SUB41_2]], 16
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = add i32 [[SHL]], [[SUB]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = add i32 [[SHL_1]], [[SUB_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = add i32 [[SHL_3]], [[SUB_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = add i32 [[SHL_2]], [[SUB_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = add i32 [[SHL18]], [[SUB12]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = add i32 [[SHL18_1]], [[SUB12_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = add i32 [[SHL18_3]], [[SUB12_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = add i32 [[SHL18_2]], [[SUB12_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = add i32 [[SHL30]], [[SUB24]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = add i32 [[SHL30_1]], [[SUB24_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = add i32 [[SHL30_3]], [[SUB24_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = add i32 [[SHL30_2]], [[SUB24_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = add i32 [[SHL42]], [[SUB36]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = add i32 [[SHL42_1]], [[SUB36_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = add i32 [[SHL42_3]], [[SUB36_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = add i32 [[SHL42_2]], [[SUB36_2]]
; UNALIGNED_VEC_MEM-NEXT: [[SUB45:%.*]] = sub i32 [[ADD_3]], [[ADD31_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_1:%.*]] = sub i32 [[ADD_2]], [[ADD31_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_3:%.*]] = sub i32 [[ADD19_3]], [[ADD43_4]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_2:%.*]] = sub i32 [[ADD19_2]], [[ADD43_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44:%.*]] = add i32 [[ADD31_3]], [[ADD_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_1:%.*]] = add i32 [[ADD31_2]], [[ADD_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_3:%.*]] = add i32 [[ADD43_4]], [[ADD19_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_2:%.*]] = add i32 [[ADD43_2]], [[ADD19_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47:%.*]] = sub i32 [[ADD31]], [[ADD43]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_1:%.*]] = sub i32 [[ADD31_1]], [[ADD43_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_3:%.*]] = sub i32 [[ADD31_4]], [[ADD43_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 2
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_2:%.*]] = sub i32 [[ADD31_5]], [[ADD43_5]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46:%.*]] = add i32 [[ADD43]], [[ADD31]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46_1:%.*]] = add i32 [[ADD43_1]], [[ADD31_1]]
@@ -427,17 +590,18 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: [[ADD103:%.*]] = add i32 [[ADD94]], [[ADD78]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD105:%.*]] = add i32 [[SUB102]], [[SUB86]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP66:%.*]] = insertelement <16 x i32> poison, i32 [[ADD46_2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP74]], <16 x i32> <i32 0, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = insertelement <16 x i32> [[TMP66]], i32 [[CONV_3]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP68:%.*]] = insertelement <16 x i32> [[TMP67]], i32 [[ADD46_1]], i32 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP69:%.*]] = insertelement <16 x i32> [[TMP68]], i32 [[ADD46]], i32 3
-; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = shufflevector <4 x i32> [[TMP49]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP69]], <16 x i32> [[TMP78]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP70:%.*]] = insertelement <16 x i32> [[TMP69]], i32 [[CONV_2]], i32 4
+; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = insertelement <16 x i32> [[TMP70]], i32 [[CONV9_2]], i32 5
; UNALIGNED_VEC_MEM-NEXT: [[TMP72:%.*]] = insertelement <16 x i32> [[TMP71]], i32 [[SUB47_1]], i32 6
; UNALIGNED_VEC_MEM-NEXT: [[TMP73:%.*]] = insertelement <16 x i32> [[TMP72]], i32 [[SUB47]], i32 7
-; UNALIGNED_VEC_MEM-NEXT: [[TMP118:%.*]] = shufflevector <4 x i32> [[TMP18]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = shufflevector <16 x i32> [[TMP73]], <16 x i32> [[TMP118]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = insertelement <16 x i32> [[TMP73]], i32 [[CONV_1]], i32 8
+; UNALIGNED_VEC_MEM-NEXT: [[TMP75:%.*]] = insertelement <16 x i32> [[TMP74]], i32 [[CONV9_1]], i32 9
+; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = insertelement <16 x i32> [[TMP75]], i32 [[CONV21_1]], i32 10
; UNALIGNED_VEC_MEM-NEXT: [[TMP77:%.*]] = insertelement <16 x i32> [[TMP76]], i32 [[ADD44]], i32 11
+; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
; UNALIGNED_VEC_MEM-NEXT: [[TMP79:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP80:%.*]] = shufflevector <16 x i32> [[TMP77]], <16 x i32> [[TMP79]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
; UNALIGNED_VEC_MEM-NEXT: [[TMP81:%.*]] = lshr <16 x i32> [[TMP80]], splat (i32 15)
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
index 3f4436f33fad6..274d9a6e950a5 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
@@ -38,18 +38,13 @@ define void @e(<4 x i16> %0) {
; THRESH-NEXT: [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP9:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; THRESH-NEXT: [[TMP13:%.*]] = icmp sgt <16 x i16> [[TMP9]], zeroinitializer
-; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; THRESH-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
-; THRESH-NEXT: [[TMP23:%.*]] = insertelement <4 x i1> poison, i1 [[TMP15]], i64 0
-; THRESH-NEXT: [[TMP16:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; THRESH-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
-; THRESH-NEXT: [[TMP24:%.*]] = insertelement <4 x i1> [[TMP23]], i1 [[TMP17]], i64 1
-; THRESH-NEXT: [[TMP18:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; THRESH-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
-; THRESH-NEXT: [[TMP22:%.*]] = insertelement <4 x i1> [[TMP24]], i1 [[TMP19]], i64 2
-; THRESH-NEXT: [[TMP20:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; THRESH-NEXT: [[TMP21:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP20]])
-; THRESH-NEXT: [[TMP25:%.*]] = insertelement <4 x i1> [[TMP22]], i1 [[TMP21]], i64 3
+; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; THRESH-NEXT: [[TMP11:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; THRESH-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP14]], [[TMP11]]
+; THRESH-NEXT: [[TMP12:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; THRESH-NEXT: [[RDX_OP1:%.*]] = or <4 x i1> [[RDX_OP]], [[TMP12]]
+; THRESH-NEXT: [[TMP15:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; THRESH-NEXT: [[TMP25:%.*]] = or <4 x i1> [[RDX_OP1]], [[TMP15]]
; THRESH-NEXT: [[TMP26]] = zext <4 x i1> [[TMP25]] to <4 x i32>
; THRESH-NEXT: br label [[VECTOR_BODY]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 0e50b135afe7f..5cb55ec2b23cb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -5,42 +5,8 @@ define <4 x i16> @test(<4 x i16> %a) {
; CHECK-LABEL: define <4 x i16> @test(
; CHECK-SAME: <4 x i16> [[A:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[A]], <4 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>, <16 x i16> [[TMP0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i16> [[TMP2]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP4]], <4 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = add <16 x i16> [[TMP5]], zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = add <16 x i16> [[TMP3]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP9:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i16> poison, i16 [[TMP9]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[TMP12:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i16> [[TMP10]], i16 [[TMP12]], i64 1
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[TMP15:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP14]])
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP13]], i16 [[TMP15]], i64 2
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: [[TMP18:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP17]])
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP18]], i64 3
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP21:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP20]])
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> poison, i16 [[TMP21]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[TMP24:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP23]])
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[TMP24]], i64 1
-; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[TMP27:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i16> [[TMP25]], i16 [[TMP27]], i64 2
-; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: [[TMP30:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP29]])
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP30]], i64 3
-; CHECK-NEXT: [[OP_RDX:%.*]] = or <4 x i16> [[TMP31]], [[TMP4]]
-; CHECK-NEXT: [[OP_RDX15:%.*]] = or <4 x i16> zeroinitializer, [[TMP19]]
-; CHECK-NEXT: [[OP_RDX16:%.*]] = or <4 x i16> [[OP_RDX]], [[OP_RDX15]]
-; CHECK-NEXT: ret <4 x i16> [[OP_RDX16]]
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, zeroinitializer
+; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
%subi = add <4 x i16> %a, %a
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 66507f647a828..2849c82142b55 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -156,18 +156,13 @@ define <4 x i1> @test6(ptr %in1, ptr %in2) {
; CHECK-NEXT: [[TMP22:%.*]] = and <16 x i1> [[TMP11]], [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <32 x i1> [[TMP6]], <32 x i1> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[TMP24:%.*]] = and <16 x i1> [[TMP22]], [[TMP23]]
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
-; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i1> poison, i1 [[TMP26]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i1> [[TMP27]], i1 [[TMP29]], i64 1
-; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[TMP32:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP31]])
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i1> [[TMP30]], i1 [[TMP32]], i64 2
-; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: [[TMP35:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP34]])
-; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i1> [[TMP33]], i1 [[TMP35]], i64 3
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP15]], [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = or <4 x i1> [[RDX_OP]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP36:%.*]] = or <4 x i1> [[RDX_OP1]], [[TMP14]]
; CHECK-NEXT: [[VBSL:%.*]] = select <4 x i1> [[TMP36]], <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: [[CMP:%.*]] = icmp ugt <4 x i32> [[VBSL]], <i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: ret <4 x i1> [[CMP]]
>From a75c3267d236ff6d8df98c6f2a0e9deef64b6feb Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Mon, 18 May 2026 22:12:43 -0700
Subject: [PATCH 2/9] Address regression with
Transforms/SLPVectorizer/RISCV/complex-loads.ll
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 52 ++-
.../SLPVectorizer/RISCV/complex-loads.ll | 314 +++++-------------
.../revec-reduced-value-vectorized-later.ll | 3 +-
3 files changed, 99 insertions(+), 270 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f7675b8683bd0..addbf0c0aa30e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29010,7 +29010,6 @@ class HorizontalReduction {
// Also, do not try to reduce const values, if the operation is not
// foldable.
bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
- R.isAnalyzedReductionRoot(EdgeInst) ||
getRdxKind(EdgeInst) != RdxKind ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
ReductionOrdering CurrentRK = IsReducedVal
@@ -29827,11 +29826,10 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered)
return VectorizedTree;
- if (!VectorValuesAndScales.empty()) {
+ if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
VectorizedTree,
- emitReduction(V, Builder, *TTI, ReductionRoot->getType()));
- }
+ emitReduction(Builder, *TTI, ReductionRoot->getType()));
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@@ -30223,9 +30221,8 @@ class HorizontalReduction {
private:
/// Creates the reduction from the given \p Vec vector value with the given
/// scale \p Scale and signedness \p IsSigned.
- Value *createSingleOp(BoUpSLP &V, IRBuilderBase &Builder,
- const TargetTransformInfo &TTI, Value *Vec,
- unsigned Scale, bool IsSigned, Type *DestTy,
+ Value *createSingleOp(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
+ Value *Vec, unsigned Scale, bool IsSigned, Type *DestTy,
bool ReducedInTree) {
Value *Rdx;
if (ReducedInTree) {
@@ -30233,26 +30230,20 @@ class HorizontalReduction {
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
- unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
Rdx = nullptr;
/*
e.g. Consider vector reduce add.
- Initial reduction is
- %add0 = add <4 x i32> zeroinitializer, %A
- %add1 = add <4 x i32> %add0, %B
-
- where,
- %A = <a, b, c, d>
- %B = <e, f, g, h>
- %add1 = A + B = <a + e, b + f, c + g, d + h>
+ RdxVal[0] = [a, b, c, d]
+ RdxVal[1] = [e, f, g, h]
+ Add0 = zeroinitializer + RdxVal[0]
+ Add1 = Add0 + RdxVal[1]
- After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
+ After revectorization with VF=2 and Vec = [a, b, c, d, e, f, g, h],
the reduction can be expressed as:
- %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
- %add0 = add <4 x i32> zeroinitializer, %A
- %add1 = add <4 x i32> %add0, %B
+ RdxVal[0] = ExtractVector(Vec, 0, 3) = [a, b, c, d]
+ RdxVal[1] = ExtractVector(Vec, 4, 7) = [e, f, g, h]
+ Add = RdxVal[0] + RdxVal[1]
*/
for (auto I : seq<unsigned>(VF)) {
auto Position = I * DestTyNumElements;
@@ -30262,8 +30253,6 @@ class HorizontalReduction {
Rdx = SubVec;
} else {
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
- if (isa<Instruction>(Rdx))
- V.analyzedReductionRoot(cast<Instruction>(Rdx));
}
}
} else {
@@ -30366,10 +30355,10 @@ class HorizontalReduction {
unsigned ScalarTyNumElements = VecTy->getNumElements();
auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
ReducedVals.size());
- for (unsigned I : seq<unsigned>(ReducedVals.size())) {
- VectorCost += TTI->getShuffleCost(TTI::SK_ExtractSubvector, VecTy,
- DstTy, {}, CostKind,
- I * ScalarTyNumElements, VecTy);
+ for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
+ VectorCost +=
+ ::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
+ CostKind, I * ScalarTyNumElements, VecTy);
VectorCost +=
TTI->getArithmeticInstrCost(RdxOpcode, VecTy, CostKind);
}
@@ -30496,13 +30485,13 @@ class HorizontalReduction {
/// Splits the values, stored in VectorValuesAndScales, into registers/free
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
- Value *emitReduction(BoUpSLP &V, IRBuilderBase &Builder,
- const TargetTransformInfo &TTI, Type *DestTy) {
+ Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
+ Type *DestTy) {
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
bool ReducedInTree) {
- Value *Rdx = createSingleOp(V, Builder, TTI, Vec, Scale, IsSigned, DestTy,
+ Value *Rdx = createSingleOp(Builder, TTI, Vec, Scale, IsSigned, DestTy,
ReducedInTree);
if (ReducedSubTree)
ReducedSubTree = createOp(Builder, RdxKind, ReducedSubTree, Rdx,
@@ -31170,6 +31159,9 @@ bool SLPVectorizerPass::vectorizeHorReduction(
Res = true;
if (auto *I = dyn_cast<Instruction>(VectorizedV); I && I != Inst) {
// Try to find another reduction.
+ if (SLPReVec && I->getType()->isVectorTy() &&
+ Inst->getType()->isVectorTy())
+ continue;
Stack.emplace(I, Level);
continue;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index ed3f85d31d314..09be099fcf709 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -277,265 +277,102 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: entry:
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20:%.*]] = getelementptr i8, ptr [[PIX1]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22:%.*]] = getelementptr i8, ptr [[PIX2]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25:%.*]] = getelementptr i8, ptr [[PIX1]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27:%.*]] = getelementptr i8, ptr [[PIX2]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34:%.*]] = getelementptr i8, ptr [[PIX2]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37:%.*]] = getelementptr i8, ptr [[PIX1]], i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39:%.*]] = getelementptr i8, ptr [[PIX2]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR644:%.*]] = getelementptr i8, ptr [[PIX2]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR_1:%.*]] = getelementptr i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX25_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX37_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 7
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX5_4:%.*]] = getelementptr i8, ptr null, i64 4
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX8_3:%.*]] = getelementptr i8, ptr null, i64 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX10_3:%.*]] = getelementptr i8, ptr null, i64 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP57:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX15_3:%.*]] = getelementptr i8, ptr null, i64 5
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX20_3:%.*]] = getelementptr i8, ptr null, i64 2
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX22_3:%.*]] = getelementptr i8, ptr null, i64 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP61:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX27_3:%.*]] = getelementptr i8, ptr null, i64 6
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX32_3:%.*]] = getelementptr i8, ptr null, i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX34_3:%.*]] = getelementptr i8, ptr null, i64 3
-; UNALIGNED_VEC_MEM-NEXT: [[ARRAYIDX39_3:%.*]] = getelementptr i8, ptr null, i64 7
-; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX37]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX25]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX13]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP60:%.*]] = load i8, ptr [[ARRAYIDX32_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX20_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX8_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = load i8, ptr [[ADD_PTR3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX34_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX22_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX10_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = load i8, ptr [[ADD_PTR644]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX34]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX22]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load i8, ptr [[ARRAYIDX10]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load i8, ptr [[PIX2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX37_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = load i8, ptr [[ARRAYIDX25_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = load i8, ptr [[ARRAYIDX13_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX3_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX39_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX15_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX5_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX39]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX15]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP32:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV_1:%.*]] = zext i8 [[TMP9]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV_3:%.*]] = zext i8 [[TMP49]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV_2:%.*]] = zext i8 [[TMP33]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9_1:%.*]] = zext i8 [[TMP8]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9_3:%.*]] = zext i8 [[TMP48]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9_2:%.*]] = zext i8 [[TMP32]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21_1:%.*]] = zext i8 [[TMP7]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21_3:%.*]] = zext i8 [[TMP47]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21_2:%.*]] = zext i8 [[TMP31]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
; UNALIGNED_VEC_MEM-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[PIX1]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = load i8, ptr [[ARRAYIDX20]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = load i8, ptr [[ARRAYIDX8]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33:%.*]] = zext i8 [[TMP50]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV21:%.*]] = zext i8 [[TMP52]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV9:%.*]] = zext i8 [[TMP53]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33_1:%.*]] = zext i8 [[TMP60]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33_3:%.*]] = zext i8 [[TMP46]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV33_2:%.*]] = zext i8 [[TMP30]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[PIX2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[TMP3]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP8:%.*]] = sub <4 x i32> [[TMP6]], [[TMP7]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP9:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP10:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP11:%.*]] = sub <4 x i32> [[TMP9]], [[TMP10]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP12:%.*]] = shl <4 x i32> [[TMP11]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP8]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP14:%.*]] = load <4 x i8>, ptr [[ADD_PTR3]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP15:%.*]] = load <4 x i8>, ptr [[ADD_PTR644]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP16:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP17:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP18:%.*]] = zext <4 x i8> [[TMP14]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP19:%.*]] = zext <4 x i8> [[TMP15]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP20:%.*]] = sub <4 x i32> [[TMP18]], [[TMP19]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP21:%.*]] = zext <4 x i8> [[TMP16]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP22:%.*]] = zext <4 x i8> [[TMP17]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP23:%.*]] = sub <4 x i32> [[TMP21]], [[TMP22]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP24:%.*]] = shl <4 x i32> [[TMP23]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP25:%.*]] = add <4 x i32> [[TMP24]], [[TMP20]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP26:%.*]] = load <4 x i8>, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP27:%.*]] = zext <4 x i8> [[TMP26]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP28:%.*]] = load <4 x i8>, ptr null, align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP29:%.*]] = zext <4 x i8> [[TMP28]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP30:%.*]] = sub <4 x i32> [[TMP27]], [[TMP29]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP106:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2:%.*]] = zext i8 [[TMP17]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2_1:%.*]] = zext i8 [[TMP13]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2_3:%.*]] = zext i8 [[TMP106]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV2_2:%.*]] = zext i8 [[TMP37]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11:%.*]] = zext i8 [[TMP16]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11_1:%.*]] = zext i8 [[TMP12]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11_3:%.*]] = zext i8 [[TMP56]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV11_2:%.*]] = zext i8 [[TMP36]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23:%.*]] = zext i8 [[TMP15]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23_1:%.*]] = zext i8 [[TMP11]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23_3:%.*]] = zext i8 [[TMP55]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV23_2:%.*]] = zext i8 [[TMP35]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35:%.*]] = zext i8 [[TMP14]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35_1:%.*]] = zext i8 [[TMP10]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35_3:%.*]] = zext i8 [[TMP54]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV35_2:%.*]] = zext i8 [[TMP34]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP59:%.*]] = zext i8 [[TMP58]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[SUB:%.*]] = sub i32 [[TMP59]], [[CONV2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB_1:%.*]] = sub i32 [[CONV_1]], [[CONV2_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB_3:%.*]] = sub i32 [[CONV_3]], [[CONV2_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB_2:%.*]] = sub i32 [[CONV_2]], [[CONV2_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12:%.*]] = sub i32 [[CONV9]], [[CONV11]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12_1:%.*]] = sub i32 [[CONV9_1]], [[CONV11_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12_3:%.*]] = sub i32 [[CONV9_3]], [[CONV11_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB12_2:%.*]] = sub i32 [[CONV9_2]], [[CONV11_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24:%.*]] = sub i32 [[CONV21]], [[CONV23]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24_1:%.*]] = sub i32 [[CONV21_1]], [[CONV23_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24_3:%.*]] = sub i32 [[CONV21_3]], [[CONV23_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB24_2:%.*]] = sub i32 [[CONV21_2]], [[CONV23_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36:%.*]] = sub i32 [[CONV33]], [[CONV35]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36_1:%.*]] = sub i32 [[CONV33_1]], [[CONV35_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36_3:%.*]] = sub i32 [[CONV33_3]], [[CONV35_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB36_2:%.*]] = sub i32 [[CONV33_2]], [[CONV35_2]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP102:%.*]] = load i8, ptr null, align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV4:%.*]] = zext i8 [[TMP5]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV4_1:%.*]] = zext i8 [[TMP21]] to i32
+; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP61]] to i32
; UNALIGNED_VEC_MEM-NEXT: [[CONV40_2:%.*]] = zext i8 [[TMP42]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV4_2:%.*]] = zext i8 [[TMP41]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14:%.*]] = zext i8 [[TMP4]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14_1:%.*]] = zext i8 [[TMP20]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14_3:%.*]] = zext i8 [[TMP57]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV14_2:%.*]] = zext i8 [[TMP40]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26:%.*]] = zext i8 [[TMP3]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26_1:%.*]] = zext i8 [[TMP19]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26_3:%.*]] = zext i8 [[TMP61]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV26_2:%.*]] = zext i8 [[TMP39]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38:%.*]] = zext i8 [[TMP51]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_1:%.*]] = zext i8 [[TMP18]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_3:%.*]] = zext i8 [[TMP102]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV38_2:%.*]] = zext i8 [[TMP38]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[TMP62:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP63:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP64:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[TMP65:%.*]] = load i8, ptr [[ARRAYIDX5_4]], align 1
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6:%.*]] = zext i8 [[TMP29]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6_1:%.*]] = zext i8 [[TMP25]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6_3:%.*]] = zext i8 [[TMP65]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV6_2:%.*]] = zext i8 [[TMP45]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16:%.*]] = zext i8 [[TMP28]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16_1:%.*]] = zext i8 [[TMP24]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16_3:%.*]] = zext i8 [[TMP64]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV16_2:%.*]] = zext i8 [[TMP44]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28:%.*]] = zext i8 [[TMP27]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28_1:%.*]] = zext i8 [[TMP23]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28_3:%.*]] = zext i8 [[TMP63]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV28_2:%.*]] = zext i8 [[TMP43]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40:%.*]] = zext i8 [[TMP26]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40_1:%.*]] = zext i8 [[TMP22]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40_3:%.*]] = zext i8 [[TMP62]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[CONV40_4:%.*]] = zext i8 [[TMP78]] to i32
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7:%.*]] = sub i32 [[CONV4]], [[CONV6]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7_1:%.*]] = sub i32 [[CONV4_1]], [[CONV6_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7_3:%.*]] = sub i32 [[CONV40_2]], [[CONV6_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB7_2:%.*]] = sub i32 [[CONV4_2]], [[CONV6_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17:%.*]] = sub i32 [[CONV14]], [[CONV16]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17_1:%.*]] = sub i32 [[CONV14_1]], [[CONV16_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17_3:%.*]] = sub i32 [[CONV14_3]], [[CONV16_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB17_2:%.*]] = sub i32 [[CONV14_2]], [[CONV16_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29:%.*]] = sub i32 [[CONV26]], [[CONV28]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29_1:%.*]] = sub i32 [[CONV26_1]], [[CONV28_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29_3:%.*]] = sub i32 [[CONV26_3]], [[CONV28_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB29_2:%.*]] = sub i32 [[CONV26_2]], [[CONV28_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41:%.*]] = sub i32 [[CONV38]], [[CONV40]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41_1:%.*]] = sub i32 [[CONV38_1]], [[CONV40_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41_3:%.*]] = sub i32 [[CONV38_3]], [[CONV40_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[SUB41_2:%.*]] = sub i32 [[CONV38_2]], [[CONV40_4]]
-; UNALIGNED_VEC_MEM-NEXT: [[SHL:%.*]] = shl i32 [[SUB7]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL_1:%.*]] = shl i32 [[SUB7_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL_3:%.*]] = shl i32 [[SUB7_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL_2:%.*]] = shl i32 [[SUB7_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18:%.*]] = shl i32 [[SUB17]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18_1:%.*]] = shl i32 [[SUB17_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18_3:%.*]] = shl i32 [[SUB17_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL18_2:%.*]] = shl i32 [[SUB17_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30:%.*]] = shl i32 [[SUB29]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30_1:%.*]] = shl i32 [[SUB29_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30_3:%.*]] = shl i32 [[SUB29_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL30_2:%.*]] = shl i32 [[SUB29_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42:%.*]] = shl i32 [[SUB41]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42_1:%.*]] = shl i32 [[SUB41_1]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42_3:%.*]] = shl i32 [[SUB41_3]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[SHL42_2:%.*]] = shl i32 [[SUB41_2]], 16
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = add i32 [[SHL]], [[SUB]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = add i32 [[SHL_1]], [[SUB_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = add i32 [[SHL_3]], [[SUB_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = add i32 [[SHL_2]], [[SUB_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = add i32 [[SHL18]], [[SUB12]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = add i32 [[SHL18_1]], [[SUB12_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = add i32 [[SHL18_3]], [[SUB12_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = add i32 [[SHL18_2]], [[SUB12_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = add i32 [[SHL30]], [[SUB24]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = add i32 [[SHL30_1]], [[SUB24_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = add i32 [[SHL30_3]], [[SUB24_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = add i32 [[SHL30_2]], [[SUB24_2]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = add i32 [[SHL42]], [[SUB36]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = add i32 [[SHL42_1]], [[SUB36_1]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = add i32 [[SHL42_3]], [[SUB36_3]]
-; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = add i32 [[SHL42_2]], [[SUB36_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP33:%.*]] = insertelement <2 x i8> poison, i8 [[TMP106]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[TMP34:%.*]] = insertelement <2 x i8> [[TMP33]], i8 [[TMP57]], i32 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP35:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_4]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP36:%.*]] = zext <4 x i8> [[TMP35]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> poison, i32 [[CONV40_2]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[TMP38:%.*]] = zext <2 x i8> [[TMP34]] to <2 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP39:%.*]] = shufflevector <2 x i32> [[TMP38]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP40:%.*]] = shufflevector <4 x i32> [[TMP37]], <4 x i32> [[TMP39]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[CONV38_3]], i32 3
+; UNALIGNED_VEC_MEM-NEXT: [[TMP58:%.*]] = sub <4 x i32> [[TMP41]], [[TMP36]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP43:%.*]] = shl <4 x i32> [[TMP58]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP44:%.*]] = add <4 x i32> [[TMP43]], [[TMP30]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP45:%.*]] = load <4 x i8>, ptr [[ADD_PTR_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP46:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_1]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP47:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP48:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP49:%.*]] = zext <4 x i8> [[TMP45]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP50:%.*]] = zext <4 x i8> [[TMP46]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP51:%.*]] = sub <4 x i32> [[TMP49]], [[TMP50]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP52:%.*]] = zext <4 x i8> [[TMP47]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP53:%.*]] = zext <4 x i8> [[TMP48]] to <4 x i32>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP54:%.*]] = sub <4 x i32> [[TMP52]], [[TMP53]]
+; UNALIGNED_VEC_MEM-NEXT: [[TMP55:%.*]] = shl <4 x i32> [[TMP54]], splat (i32 16)
+; UNALIGNED_VEC_MEM-NEXT: [[TMP56:%.*]] = add <4 x i32> [[TMP55]], [[TMP51]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_3:%.*]] = extractelement <4 x i32> [[TMP13]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45:%.*]] = sub i32 [[ADD_3]], [[ADD31_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_2:%.*]] = extractelement <4 x i32> [[TMP25]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_1:%.*]] = sub i32 [[ADD_2]], [[ADD31_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_3:%.*]] = sub i32 [[ADD19_3]], [[ADD43_4]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD19_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 0
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_2:%.*]] = extractelement <4 x i32> [[TMP56]], i32 1
; UNALIGNED_VEC_MEM-NEXT: [[SUB45_2:%.*]] = sub i32 [[ADD19_2]], [[ADD43_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44:%.*]] = add i32 [[ADD31_3]], [[ADD_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_1:%.*]] = add i32 [[ADD31_2]], [[ADD_2]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_3:%.*]] = add i32 [[ADD43_4]], [[ADD19_3]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD44_2:%.*]] = add i32 [[ADD43_2]], [[ADD19_2]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47:%.*]] = sub i32 [[ADD31]], [[ADD43]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_1:%.*]] = extractelement <4 x i32> [[TMP25]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_1:%.*]] = sub i32 [[ADD31_1]], [[ADD43_1]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_4:%.*]] = extractelement <4 x i32> [[TMP44]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_3:%.*]] = extractelement <4 x i32> [[TMP44]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_3:%.*]] = sub i32 [[ADD31_4]], [[ADD43_3]]
+; UNALIGNED_VEC_MEM-NEXT: [[ADD31_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 2
+; UNALIGNED_VEC_MEM-NEXT: [[ADD43_5:%.*]] = extractelement <4 x i32> [[TMP56]], i32 3
; UNALIGNED_VEC_MEM-NEXT: [[SUB47_2:%.*]] = sub i32 [[ADD31_5]], [[ADD43_5]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46:%.*]] = add i32 [[ADD43]], [[ADD31]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD46_1:%.*]] = add i32 [[ADD43_1]], [[ADD31_1]]
@@ -590,18 +427,17 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; UNALIGNED_VEC_MEM-NEXT: [[ADD103:%.*]] = add i32 [[ADD94]], [[ADD78]]
; UNALIGNED_VEC_MEM-NEXT: [[ADD105:%.*]] = add i32 [[SUB102]], [[SUB86]]
; UNALIGNED_VEC_MEM-NEXT: [[TMP66:%.*]] = insertelement <16 x i32> poison, i32 [[ADD46_2]], i32 0
-; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = insertelement <16 x i32> [[TMP66]], i32 [[CONV_3]], i32 1
+; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP74]], <16 x i32> <i32 0, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP68:%.*]] = insertelement <16 x i32> [[TMP67]], i32 [[ADD46_1]], i32 2
; UNALIGNED_VEC_MEM-NEXT: [[TMP69:%.*]] = insertelement <16 x i32> [[TMP68]], i32 [[ADD46]], i32 3
-; UNALIGNED_VEC_MEM-NEXT: [[TMP70:%.*]] = insertelement <16 x i32> [[TMP69]], i32 [[CONV_2]], i32 4
-; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = insertelement <16 x i32> [[TMP70]], i32 [[CONV9_2]], i32 5
+; UNALIGNED_VEC_MEM-NEXT: [[TMP78:%.*]] = shufflevector <4 x i32> [[TMP49]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP69]], <16 x i32> [[TMP78]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP72:%.*]] = insertelement <16 x i32> [[TMP71]], i32 [[SUB47_1]], i32 6
; UNALIGNED_VEC_MEM-NEXT: [[TMP73:%.*]] = insertelement <16 x i32> [[TMP72]], i32 [[SUB47]], i32 7
-; UNALIGNED_VEC_MEM-NEXT: [[TMP74:%.*]] = insertelement <16 x i32> [[TMP73]], i32 [[CONV_1]], i32 8
-; UNALIGNED_VEC_MEM-NEXT: [[TMP75:%.*]] = insertelement <16 x i32> [[TMP74]], i32 [[CONV9_1]], i32 9
-; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = insertelement <16 x i32> [[TMP75]], i32 [[CONV21_1]], i32 10
+; UNALIGNED_VEC_MEM-NEXT: [[TMP118:%.*]] = shufflevector <4 x i32> [[TMP18]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_VEC_MEM-NEXT: [[TMP76:%.*]] = shufflevector <16 x i32> [[TMP73]], <16 x i32> [[TMP118]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP77:%.*]] = insertelement <16 x i32> [[TMP76]], i32 [[ADD44]], i32 11
-; UNALIGNED_VEC_MEM-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
; UNALIGNED_VEC_MEM-NEXT: [[TMP79:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; UNALIGNED_VEC_MEM-NEXT: [[TMP80:%.*]] = shufflevector <16 x i32> [[TMP77]], <16 x i32> [[TMP79]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
; UNALIGNED_VEC_MEM-NEXT: [[TMP81:%.*]] = lshr <16 x i32> [[TMP80]], splat (i32 15)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 5cb55ec2b23cb..4398ace751548 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -5,7 +5,8 @@ define <4 x i16> @test(<4 x i16> %a) {
; CHECK-LABEL: define <4 x i16> @test(
; CHECK-SAME: <4 x i16> [[A:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, zeroinitializer
+; CHECK-NEXT: [[SUBI:%.*]] = add <4 x i16> zeroinitializer, zeroinitializer
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, [[SUBI]]
; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
>From 318b5b8d007af6ca5a4dbb41a48269b59c73f848 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Mon, 25 May 2026 00:56:01 -0700
Subject: [PATCH 3/9] address reviewer's comments
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 28 +++++++++++--------
.../revec-reduced-value-vectorized-later.ll | 3 +-
2 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index addbf0c0aa30e..bb887acc28778 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3877,11 +3877,7 @@ class slpvectorizer::BoUpSLP {
AnalyzedReductionVals.insert(hash_value(VL));
}
/// Clear the list of the analyzed reduction root instructions.
- void clearReductionData() {
- AnalyzedReductionsRoots.clear();
- AnalyzedReductionVals.clear();
- AnalyzedMinBWVals.clear();
- }
+ void clearReductionData();
/// Checks if the given value is gathered in one of the nodes.
bool isAnyGathered(const SmallDenseSet<Value *> &Vals) const {
return any_of(MustGather, [&](Value *V) { return Vals.contains(V); });
@@ -28460,6 +28456,8 @@ class HorizontalReduction {
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
+ /// List of values to ignore while collecting reduction values.
+ inline static SmallDenseSet<Value *> IgnoredVals;
/// Maps reduced value to the corresponding reduction operation.
SmallDenseMap<Value *, SmallVector<Instruction *>, 16> ReducedValsToOps;
WeakTrackingVH ReductionRoot;
@@ -29028,6 +29026,8 @@ class HorizontalReduction {
Operands.contains(EdgeInst) ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>))) {
+ if (IgnoredVals.contains(EdgeVal))
+ continue;
PossibleReducedVals.push_back(EdgeVal);
if (EdgeInst && !isCmpSelMinMax(EdgeInst))
Operands.insert_range(EdgeInst->operands());
@@ -29035,7 +29035,8 @@ class HorizontalReduction {
}
if (CurrentRK == ReductionOrdering::Ordered)
RK = ReductionOrdering::Ordered;
- ReductionOps.push_back(EdgeInst);
+ if (!IgnoredVals.contains(EdgeVal))
+ ReductionOps.push_back(EdgeInst);
}
};
// Try to regroup reduced values so that it gets more profitable to try to
@@ -30249,11 +30250,13 @@ class HorizontalReduction {
auto Position = I * DestTyNumElements;
Value *SubVec =
createExtractVector(Builder, Vec, DestTyNumElements, Position);
+ IgnoredVals.insert(SubVec);
if (!Rdx) {
Rdx = SubVec;
} else {
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
}
+ IgnoredVals.insert(Rdx);
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
@@ -30353,8 +30356,6 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
- auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
- ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
@@ -30865,6 +30866,14 @@ class HorizontalReduction {
static RecurKind getRdxKind(Value *V) {
return HorizontalReduction::getRdxKind(V);
}
+
+void BoUpSLP::clearReductionData() {
+ AnalyzedReductionsRoots.clear();
+ AnalyzedReductionVals.clear();
+ AnalyzedMinBWVals.clear();
+ HorizontalReduction::clearReductionData();
+}
+
static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) {
if (auto *IE = dyn_cast<InsertElementInst>(InsertInst))
return cast<FixedVectorType>(IE->getType())->getNumElements();
@@ -31159,9 +31168,6 @@ bool SLPVectorizerPass::vectorizeHorReduction(
Res = true;
if (auto *I = dyn_cast<Instruction>(VectorizedV); I && I != Inst) {
// Try to find another reduction.
- if (SLPReVec && I->getType()->isVectorTy() &&
- Inst->getType()->isVectorTy())
- continue;
Stack.emplace(I, Level);
continue;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 4398ace751548..5cb55ec2b23cb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -5,8 +5,7 @@ define <4 x i16> @test(<4 x i16> %a) {
; CHECK-LABEL: define <4 x i16> @test(
; CHECK-SAME: <4 x i16> [[A:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[SUBI:%.*]] = add <4 x i16> zeroinitializer, zeroinitializer
-; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, [[SUBI]]
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, zeroinitializer
; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
>From b5fa249cd05252ec4ed304fc205c121cbe973093 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 2 Jun 2026 04:07:10 -0700
Subject: [PATCH 4/9] address reviewer's comments
Reject reduction tree with 3 nodes.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 40 +++++++++++--------
llvm/test/Transforms/SLPVectorizer/revec.ll | 29 ++++++++++++++
2 files changed, 52 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bb887acc28778..74d04d945b0c6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3877,7 +3877,11 @@ class slpvectorizer::BoUpSLP {
AnalyzedReductionVals.insert(hash_value(VL));
}
/// Clear the list of the analyzed reduction root instructions.
- void clearReductionData();
+ void clearReductionData() {
+ AnalyzedReductionsRoots.clear();
+ AnalyzedReductionVals.clear();
+ AnalyzedMinBWVals.clear();
+ }
/// Checks if the given value is gathered in one of the nodes.
bool isAnyGathered(const SmallDenseSet<Value *> &Vals) const {
return any_of(MustGather, [&](Value *V) { return Vals.contains(V); });
@@ -18041,6 +18045,21 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
if (!DebugCounter::shouldExecute(VectorizedGraphs))
return true;
+ // If we are revectorizing reduction, it may result in same reduction pattern
+ // with shuffles as leaves of the reduction. Prevent SLP from revectorizing
+ // that shuffle pattern.
+ if (SLPReVec && ForReduction && VectorizableTree.size() == 3 &&
+ VectorizableTree[0]->State == TreeEntry::Vectorize &&
+ VectorizableTree[0]->getOpcode() == Instruction::ShuffleVector &&
+ VectorizableTree[1]->isGather() &&
+ isSplat(VectorizableTree[1]->Scalars) &&
+ VectorizableTree[2]->isGather() &&
+ allConstant(VectorizableTree[2]->Scalars)) {
+ LLVM_DEBUG(dbgs() << "SLP: Rejecting reduction tree with 3 nodes(shuffle "
+ "as root and remaining are gather nodes).\n");
+ return true;
+ }
+
// Graph is empty - do nothing.
if (VectorizableTree.empty()) {
assert(ExternalUses.empty() && "We shouldn't have any external users");
@@ -28456,8 +28475,6 @@ class HorizontalReduction {
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
- /// List of values to ignore while collecting reduction values.
- inline static SmallDenseSet<Value *> IgnoredVals;
/// Maps reduced value to the corresponding reduction operation.
SmallDenseMap<Value *, SmallVector<Instruction *>, 16> ReducedValsToOps;
WeakTrackingVH ReductionRoot;
@@ -29026,8 +29043,6 @@ class HorizontalReduction {
Operands.contains(EdgeInst) ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>))) {
- if (IgnoredVals.contains(EdgeVal))
- continue;
PossibleReducedVals.push_back(EdgeVal);
if (EdgeInst && !isCmpSelMinMax(EdgeInst))
Operands.insert_range(EdgeInst->operands());
@@ -29035,8 +29050,7 @@ class HorizontalReduction {
}
if (CurrentRK == ReductionOrdering::Ordered)
RK = ReductionOrdering::Ordered;
- if (!IgnoredVals.contains(EdgeVal))
- ReductionOps.push_back(EdgeInst);
+ ReductionOps.push_back(EdgeInst);
}
};
// Try to regroup reduced values so that it gets more profitable to try to
@@ -30250,13 +30264,11 @@ class HorizontalReduction {
auto Position = I * DestTyNumElements;
Value *SubVec =
createExtractVector(Builder, Vec, DestTyNumElements, Position);
- IgnoredVals.insert(SubVec);
if (!Rdx) {
Rdx = SubVec;
} else {
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
}
- IgnoredVals.insert(Rdx);
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
@@ -30356,6 +30368,8 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
+ auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
+ ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
@@ -30866,14 +30880,6 @@ class HorizontalReduction {
static RecurKind getRdxKind(Value *V) {
return HorizontalReduction::getRdxKind(V);
}
-
-void BoUpSLP::clearReductionData() {
- AnalyzedReductionsRoots.clear();
- AnalyzedReductionVals.clear();
- AnalyzedMinBWVals.clear();
- HorizontalReduction::clearReductionData();
-}
-
static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) {
if (auto *IE = dyn_cast<InsertElementInst>(InsertInst))
return cast<FixedVectorType>(IE->getType())->getNumElements();
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 2849c82142b55..8a813d94af68b 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -469,3 +469,32 @@ entry:
store <4 x float> %11, ptr %1, align 16
ret i32 0
}
+
+define <4 x i32> @hor_reduction_four_points(ptr %a) {
+; CHECK-LABEL: @hor_reduction_four_points(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = add <4 x i32> [[RDX_OP]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[RDX_OP2:%.*]] = add <4 x i32> [[RDX_OP1]], [[TMP4]]
+; CHECK-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP2]], zeroinitializer
+; CHECK-NEXT: ret <4 x i32> [[OP_RDX]]
+;
+entry:
+ %gep1 = getelementptr <4 x i32>, ptr %a, i64 1
+ %gep2 = getelementptr <4 x i32>, ptr %a, i64 2
+ %gep3 = getelementptr <4 x i32>, ptr %a, i64 3
+ %a0 = load <4 x i32>, ptr %a
+ %a1 = load <4 x i32>, ptr %gep1
+ %a2 = load <4 x i32>, ptr %gep2
+ %a3 = load <4 x i32>, ptr %gep3
+ %add0 = add <4 x i32> zeroinitializer, %a0
+ %add1 = add <4 x i32> %add0, %a1
+ %add2 = add <4 x i32> %add1, %a2
+ %add3 = add <4 x i32> %add2, %a3
+ ret <4 x i32> %add3
+}
>From 796b5dba47adb802f8e4fbb6567b63bc749241ac Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Wed, 3 Jun 2026 08:47:48 -0700
Subject: [PATCH 5/9] address unused variable error
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 74d04d945b0c6..1d00310883a45 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30368,8 +30368,6 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
- auto *DstTy = FixedVectorType::get(VecTy->getScalarType(),
- ReducedVals.size());
for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
>From bb71921712e22ef07fa227dfb3a9a67b90da7df5 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 4 Jun 2026 01:33:51 -0700
Subject: [PATCH 6/9] address formatting issues
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 37 ++++++++++---------
1 file changed, 20 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1d00310883a45..da92d755df246 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30245,30 +30245,33 @@ class HorizontalReduction {
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
+ // e.g. Consider vector reduce add.
+ // Initial reduction is
+ // clang-format off
+ // %add0 = add <4 x i32> zeroinitializer, %A
+ // %add1 = add <4 x i32> %add0, %B
+ //
+ // where,
+ // %A = <a, b, c, d>
+ // %B = <e, f, g, h>
+ // %add1 = A + B = <a + e, b + f, c + g, d + h>
+ //
+ // After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
+ // the reduction can be expressed as:
+ // %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ // %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ // %add0 = add <4 x i32> zeroinitializer, %A
+ // %add1 = add <4 x i32> %add0, %B
+ // clang-format on
Rdx = nullptr;
- /*
- e.g. Consider vector reduce add.
-
- RdxVal[0] = [a, b, c, d]
- RdxVal[1] = [e, f, g, h]
- Add0 = zeroinitializer + RdxVal[0]
- Add1 = Add0 + RdxVal[1]
-
- After revectorization with VF=2 and Vec = [a, b, c, d, e, f, g, h],
- the reduction can be expressed as:
- RdxVal[0] = ExtractVector(Vec, 0, 3) = [a, b, c, d]
- RdxVal[1] = ExtractVector(Vec, 4, 7) = [e, f, g, h]
- Add = RdxVal[0] + RdxVal[1]
- */
for (auto I : seq<unsigned>(VF)) {
auto Position = I * DestTyNumElements;
Value *SubVec =
createExtractVector(Builder, Vec, DestTyNumElements, Position);
- if (!Rdx) {
+ if (!Rdx)
Rdx = SubVec;
- } else {
+ else
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
- }
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
>From b494e37f6965d813b1ee8109ddd10da47a495ba5 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 4 Jun 2026 19:26:20 -0700
Subject: [PATCH 7/9] consider FMF in arithmetic instruction cost and other
minor issues addressed
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 14 +++++++++++---
llvm/test/Transforms/SLPVectorizer/revec.ll | 4 ++--
2 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index da92d755df246..ad05803612791 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30371,12 +30371,20 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
- for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
+ for (unsigned I : seq<unsigned>(ReducedVals.size())) {
VectorCost +=
::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
CostKind, I * ScalarTyNumElements, VecTy);
- VectorCost +=
- TTI->getArithmeticInstrCost(RdxOpcode, VecTy, CostKind);
+ }
+ // We get one less arithmetic instruction compared to number of
+ // reduced values. We are also passing CtxI so that the backend can
+ // synthesize FMF from the respective reduction instruction.
+ for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
+ VectorCost += TTI->getArithmeticInstrCost(
+ RdxOpcode, VecTy, CostKind,
+ TTI::getOperandInfo(ReducedVals[I]),
+ TTI::getOperandInfo(ReducedVals[I + 1]), {},
+ ReducedValsToOps[ReducedVals[I]].front());
}
} else {
Type *RedTy = VectorTy->getElementType();
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 8a813d94af68b..346669c53824f 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -481,7 +481,7 @@ define <4 x i32> @hor_reduction_four_points(ptr %a) {
; CHECK-NEXT: [[RDX_OP1:%.*]] = add <4 x i32> [[RDX_OP]], [[TMP3]]
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[RDX_OP2:%.*]] = add <4 x i32> [[RDX_OP1]], [[TMP4]]
-; CHECK-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP2]], zeroinitializer
+; CHECK-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP2]], <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <4 x i32> [[OP_RDX]]
;
entry:
@@ -492,7 +492,7 @@ entry:
%a1 = load <4 x i32>, ptr %gep1
%a2 = load <4 x i32>, ptr %gep2
%a3 = load <4 x i32>, ptr %gep3
- %add0 = add <4 x i32> zeroinitializer, %a0
+ %add0 = add <4 x i32> <i32 0, i32 1, i32 2, i32 3>, %a0
%add1 = add <4 x i32> %add0, %a1
%add2 = add <4 x i32> %add1, %a2
%add3 = add <4 x i32> %add2, %a3
>From b7368924c4d970ee07fb4465fd088500e4c18f88 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 9 Jun 2026 17:47:51 -0700
Subject: [PATCH 8/9] Make scalar cost computation better by passing the
reduction instruction to scalar cost compute function
Passing reduction instruction should help getArithmeticInstrCost in synthesizing FMF from the instruction
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 105 ++++++++++--------
1 file changed, 56 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index ad05803612791..8a2e5a97f7573 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30300,57 +30300,61 @@ class HorizontalReduction {
// If all of the reduced values are constant, the vector cost is 0, since
// the reduction value can be calculated at the compile time.
bool AllConsts = allConstant(ReducedVals);
- auto EvaluateScalarCost = [&](function_ref<InstructionCost()> GenCostFn) {
- InstructionCost Cost = 0;
- // Scalar cost is repeated for N-1 elements.
- int Cnt = ReducedVals.size();
- for (Value *RdxVal : ReducedVals) {
- if (!isa<Instruction>(RdxVal))
- continue;
- if (Cnt == 1) {
- unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
- Cost += (SameValueCount ? SameValueCount - 1 : 0) * GenCostFn();
- break;
- }
- --Cnt;
- if (RdxVal->hasNUsesOrMore(IsCmpSelMinMax ? 3 : 2)) {
- unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
- Cost += (SameValueCount ? SameValueCount : 1) * GenCostFn();
- continue;
- }
- InstructionCost ScalarCost = 0;
- for (User *U : RdxVal->users()) {
- auto *RdxOp = cast<Instruction>(U);
- if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
- if (RdxKind == RecurKind::FAdd) {
- InstructionCost FMACost = canConvertToFMA(
- RdxOp, getSameOpcode(RdxOp, TLI), DT, DL, *TTI, TLI);
- if (FMACost.isValid()) {
- LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
- if (auto *I = dyn_cast<Instruction>(RdxVal)) {
- // Also, exclude scalar fmul cost.
- InstructionCost FMulCost =
- TTI->getInstructionCost(I, CostKind);
- LLVM_DEBUG(dbgs() << "Minus FMul cost: " << FMulCost << "\n");
- FMACost -= FMulCost;
+ auto EvaluateScalarCost =
+ [&](function_ref<InstructionCost(Instruction *)> GenCostFn) {
+ InstructionCost Cost = 0;
+ // Scalar cost is repeated for N-1 elements.
+ int Cnt = ReducedVals.size();
+ for (auto [Idx, RdxVal] : enumerate(ReducedVals)) {
+ if (!isa<Instruction>(RdxVal))
+ continue;
+ Instruction *RdxOp = ReducedValsToOps[ReducedVals[Idx]].front();
+ if (Cnt == 1) {
+ unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
+ Cost +=
+ (SameValueCount ? SameValueCount - 1 : 0) * GenCostFn(RdxOp);
+ break;
+ }
+ --Cnt;
+ if (RdxVal->hasNUsesOrMore(IsCmpSelMinMax ? 3 : 2)) {
+ unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
+ Cost += (SameValueCount ? SameValueCount : 1) * GenCostFn(RdxOp);
+ continue;
+ }
+ InstructionCost ScalarCost = 0;
+ for (User *U : RdxVal->users()) {
+ auto *RdxOp = cast<Instruction>(U);
+ if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
+ if (RdxKind == RecurKind::FAdd) {
+ InstructionCost FMACost = canConvertToFMA(
+ RdxOp, getSameOpcode(RdxOp, TLI), DT, DL, *TTI, TLI);
+ if (FMACost.isValid()) {
+ LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
+ if (auto *I = dyn_cast<Instruction>(RdxVal)) {
+ // Also, exclude scalar fmul cost.
+ InstructionCost FMulCost =
+ TTI->getInstructionCost(I, CostKind);
+ LLVM_DEBUG(dbgs()
+ << "Minus FMul cost: " << FMulCost << "\n");
+ FMACost -= FMulCost;
+ }
+ ScalarCost += FMACost;
+ continue;
+ }
}
- ScalarCost += FMACost;
+ ScalarCost += TTI->getInstructionCost(RdxOp, CostKind);
continue;
}
+ ScalarCost = InstructionCost::getInvalid();
+ break;
}
- ScalarCost += TTI->getInstructionCost(RdxOp, CostKind);
- continue;
+ if (ScalarCost.isValid())
+ Cost += ScalarCost;
+ else
+ Cost += GenCostFn(RdxOp);
}
- ScalarCost = InstructionCost::getInvalid();
- break;
- }
- if (ScalarCost.isValid())
- Cost += ScalarCost;
- else
- Cost += GenCostFn();
- }
- return Cost;
- };
+ return Cost;
+ };
// Require reduction cost if:
// 1. This type is not a full register type and no other vectors with the
// same type in the storage (first vector with small type).
@@ -30454,8 +30458,11 @@ class HorizontalReduction {
}
}
}
- ScalarCost = EvaluateScalarCost([&]() {
- return TTI->getArithmeticInstrCost(RdxOpcode, ScalarTy, CostKind);
+ ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
+ return TTI->getArithmeticInstrCost(
+ RdxOpcode, ScalarTy, CostKind,
+ TTI::getOperandInfo(RdxOp->getOperand(0)),
+ TTI::getOperandInfo(RdxOp->getOperand(1)), {}, RdxOp);
});
break;
}
@@ -30490,7 +30497,7 @@ class HorizontalReduction {
}
}
}
- ScalarCost = EvaluateScalarCost([&]() {
+ ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
IntrinsicCostAttributes ICA(Id, ScalarTy, {ScalarTy, ScalarTy}, FMF);
return TTI->getIntrinsicInstrCost(ICA, CostKind);
});
>From 59c62320c157a1b035a585d1ce956af3c0e648d7 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 9 Jun 2026 18:55:36 -0700
Subject: [PATCH 9/9] Rebase
---
.../revec-reduced-value-vectorized-later.ll | 31 ++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 5cb55ec2b23cb..c69e07b77affa 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -5,7 +5,36 @@ define <4 x i16> @test(<4 x i16> %a) {
; CHECK-LABEL: define <4 x i16> @test(
; CHECK-SAME: <4 x i16> [[A:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> zeroinitializer, zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[A]], <4 x i16> poison, <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <24 x i16> <i16 0, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <24 x i16> [[TMP0]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[TMP2:%.*]] = add <24 x i16> [[TMP1]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <24 x i16> [[TMP2]], <24 x i16> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <48 x i16> [[TMP3]], <48 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP5:%.*]] = add <48 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i16> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = or <4 x i16> [[RDX_OP]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[RDX_OP2:%.*]] = or <4 x i16> [[RDX_OP1]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[RDX_OP3:%.*]] = or <4 x i16> [[RDX_OP2]], [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[RDX_OP4:%.*]] = or <4 x i16> [[RDX_OP3]], [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; CHECK-NEXT: [[RDX_OP5:%.*]] = or <4 x i16> [[RDX_OP4]], [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[RDX_OP6:%.*]] = or <4 x i16> [[RDX_OP5]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEXT: [[RDX_OP7:%.*]] = or <4 x i16> [[RDX_OP6]], [[TMP14]]
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT: [[RDX_OP8:%.*]] = or <4 x i16> [[RDX_OP7]], [[TMP15]]
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; CHECK-NEXT: [[RDX_OP9:%.*]] = or <4 x i16> [[RDX_OP8]], [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT: [[RDX_OP10:%.*]] = or <4 x i16> [[RDX_OP9]], [[TMP17]]
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> [[RDX_OP10]], [[TMP4]]
; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
More information about the llvm-commits
mailing list