[llvm] [SLP]Vectorize consecutive loads with undef lanes as a wide load (PR #228872)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 4 05:07:50 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/228872
>From fe2d13640b3280a643241c795e3d0345c3852668 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 4 Oct 2026 04:31:40 -0700
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 132 +++++++++++++++---
.../SLPVectorizer/SLPMemoryUtils.cpp | 54 +++++++
.../Vectorize/SLPVectorizer/SLPMemoryUtils.h | 16 +++
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 15 ++
.../Vectorize/SLPVectorizer/SLPUtils.h | 9 ++
.../SLPVectorizer/SystemZ/pr34619.ll | 7 +-
.../X86/absorbing-copyable-lane.ll | 118 +++++-----------
.../X86/wide-load-absorbed-lane.ll | 98 +++++--------
8 files changed, 285 insertions(+), 164 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c105f41bc15ab..3dce3946841cb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6304,6 +6304,12 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
if (DL->getTypeSizeInBits(ScalarTy) != DL->getTypeAllocSizeInBits(ScalarTy))
return LoadsState::Gather;
+ // The undef lanes are covered by a single wide load of the consecutive loads.
+ if (any_of(VL, IsaPred<UndefValue>))
+ return canWidenLoadsOverUndefLanes(VL, *DL, *SE, *AC, *DT, *TLI)
+ ? LoadsState::Vectorize
+ : LoadsState::Gather;
+
// Make sure all loads in the bundle are simple - we can't vectorize
// atomic or volatile loads.
PointerOps.clear();
@@ -10758,9 +10764,11 @@ class InstructionsCompatibilityAnalysis {
/// instruction and its actual operands should be returned, or it is a
/// copyable element and its should be represented as idempotent instruction.
/// \p SelfOp selects the op(V, V) modeling for a copyable lane instead of
- /// op(V, identity).
+ /// op(V, identity). \p UndefPlaceholder selects op(V, undef) for the
+ /// absorbing constant V.
SmallVector<Value *> getOperands(const InstructionsState &S, Value *V,
- bool SelfOp) const {
+ bool SelfOp,
+ bool UndefPlaceholder = false) const {
if (SelfOp)
return {V, V};
if (isa<PoisonValue>(V)) {
@@ -10802,9 +10810,37 @@ class InstructionsCompatibilityAnalysis {
return {ConstantExpr::getTrunc(cast<Constant>(V),
cast<CastInst>(MainOp)->getSrcTy())};
assert(isSupportedMainOp(MainOp) && "Unsupported opcode");
+ Type *Ty = V->getType();
+ if (UndefPlaceholder && V == ConstantExpr::getBinOpAbsorber(MainOpcode, Ty))
+ return {V, UndefValue::get(Ty)};
+ if (isa<UndefValue>(V) && isUndefTolerantBinOp(MainOp))
+ return {V, V};
return {V, selectBestIdempotentValue()};
}
+ /// Checks if the lanes with the absorbing constant may use undef as the other
+ /// operand: the other operands of the remaining lanes are the binary
+ /// operators of the different loads, whose undef lane is covered by the wide
+ /// load.
+ bool canUseUndefPlaceholders(const InstructionsState &S,
+ ArrayRef<Value *> VL) const {
+ Constant *Absorber =
+ ConstantExpr::getBinOpAbsorber(MainOpcode, VL.front()->getType());
+ if (!Absorber || !is_contained(VL, Absorber))
+ return false;
+ SmallPtrSet<const LoadInst *, 8> Loads;
+ return all_of(VL, [&](Value *V) {
+ if (isa<PoisonValue>(V) || S.isCopyableElement(V))
+ return true;
+ SmallVector<Value *> Ops = convertTo(cast<Instruction>(V), S).second;
+ return count_if(Ops, IsaPred<Constant>) == 1 &&
+ any_of(Ops, [&](Value *Op) {
+ const LoadInst *LI = getLoadOfUndefTolerantBinOp(Op);
+ return LI && Loads.insert(LI).second;
+ });
+ });
+ }
+
/// Returns the copyable lanes of an idempotent binop to be modeled as
/// op(V, V) == V instead of op(V, identity): when the operand column the
/// lane joins also holds a constant lane, the reorder would align that
@@ -11278,6 +11314,18 @@ class InstructionsCompatibilityAnalysis {
if (SelectOp)
return InstructionsState(SelectOp, SelectOp);
}
+ // The undef lanes among the loads are the copyable elements, covered by a
+ // single wide load.
+ if (!S && VectorizeCopyableElements &&
+ all_of(VL, IsaPred<LoadInst, UndefValue>) && any_of(VL, [](Value *V) {
+ return isa<UndefValue>(V) && !isa<PoisonValue>(V);
+ })) {
+ SmallVector<Value *> Loads(make_isa_range<LoadInst>(VL));
+ if (Loads.size() > 1 && allSameBlock(Loads)) {
+ auto *LI = cast<LoadInst>(Loads.front());
+ return InstructionsState(LI, LI, /*HasCopyables=*/true);
+ }
+ }
if (S && S.isAltShuffle()) {
Type *ScalarTy = S.getMainOp()->getType();
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
@@ -11548,6 +11596,11 @@ class InstructionsCompatibilityAnalysis {
BoUpSLP::ValueList(VL.size(), MainOp->getOperand(0)));
return Operands;
}
+ // The undef lanes of the loads have no pointer operands.
+ if (MainOpcode == Instruction::Load) {
+ buildOriginalOperands(S, VL, Operands);
+ return Operands;
+ }
// Excludes the trailing callee operand (2 for min/max, 3 for fmuladd).
// getNumberOfPotentiallyCommutativeOps collapses fmuladd to 2 and must
// not be used here. Only the 2-operand case is commutative-normalized.
@@ -11559,10 +11612,11 @@ class InstructionsCompatibilityAnalysis {
Operands.assign(NumMainOpOperands,
BoUpSLP::ValueList(VL.size(), nullptr));
SmallPtrSet<const Value *, 4> SelfOpLanes = findSelfOpLanes(S, VL, R);
+ const bool UndefPlaceholders = canUseUndefPlaceholders(S, VL);
// Populate operands for every lane.
for (auto [Idx, V] : enumerate(VL)) {
SmallVector<Value *> OperandsForValue =
- getOperands(S, V, SelfOpLanes.contains(V));
+ getOperands(S, V, SelfOpLanes.contains(V), UndefPlaceholders);
for (auto [OperandIdx, Operand] : enumerate(OperandsForValue))
Operands[OperandIdx][Idx] = Operand;
}
@@ -14744,7 +14798,7 @@ void BoUpSLP::transformNodes() {
case Instruction::Load: {
// No need to reorder masked gather loads, just reorder the scalar
// operands.
- if (E.State != TreeEntry::Vectorize)
+ if (E.State != TreeEntry::Vectorize || E.hasCopyableElements())
break;
Type *ScalarTy = E.getMainOp()->getType();
auto *VecTy =
@@ -17623,7 +17677,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
} else {
VecLdCost = TTI->getMemoryOpCost(
- Instruction::Load, VecTy, LI0->getAlign(),
+ Instruction::Load, VecTy,
+ commonAlignment(LI0->getAlign(), getVectorLoadShift(E->Scalars)),
LI0->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI0->getPointerOperand()));
}
@@ -17726,9 +17781,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return Cost;
// Estimate cost of GEPs since this tree node is a terminator.
- SmallVector<Value *> PointerOps(VL.size());
- for (auto [I, V] : enumerate(VL))
- PointerOps[I] = cast<LoadInst>(V)->getPointerOperand();
+ SmallVector<Value *> PointerOps;
+ for (LoadInst *LI : make_isa_range<LoadInst>(VL))
+ PointerOps.push_back(LI->getPointerOperand());
return Cost + GetGEPCostDiff(PointerOps, LI0->getPointerOperand());
}
case Instruction::Store: {
@@ -24008,18 +24063,29 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if (!AllNoInfs)
I->setHasNoInfs(false);
}
- // Copyable lanes modeled as op(V, V) do not preserve the disjoint flag:
- // or disjoint(V, V) is poison unless V is zero. Matched by value, so
- // operand reordering does not affect the detection.
+ // Copyable lanes modeled as op(V, V) or op(V, undef) do not preserve the
+ // disjoint flag: or disjoint(V, V) is poison unless V is zero, and the
+ // undef operand may be any value. Matched by value, so operand reordering
+ // does not affect the detection.
auto *PDI = dyn_cast<PossiblyDisjointInst>(I);
if (E->hasCopyableElements() && PDI &&
any_of(enumerate(E->Scalars), [&](const auto &P) {
Value *Scalar = P.value();
+ auto IsScalarOrUndef = [Scalar](Value *Op) {
+ return Op == Scalar || isa<UndefValue>(Op);
+ };
return E->isCopyableElement(Scalar) && !match(Scalar, m_Zero()) &&
- E->getOperand(0)[P.index()] == Scalar &&
- E->getOperand(1)[P.index()] == Scalar;
+ IsScalarOrUndef(E->getOperand(0)[P.index()]) &&
+ IsScalarOrUndef(E->getOperand(1)[P.index()]);
}))
PDI->setIsDisjoint(false);
+ // The undef copyable lanes are modeled as op(undef, undef), which must not
+ // overflow into poison.
+ if (E->hasCopyableElements() && isUndefTolerantBinOp(I) &&
+ any_of(E->Scalars, [&](Value *Scalar) {
+ return isa<UndefValue>(Scalar) && E->isCopyableElement(Scalar);
+ }))
+ I->dropPoisonGeneratingFlags();
// Drop nuw flags for abs(sub(commutative), true).
if (!MinBWs.contains(E) && Opcode == Instruction::Sub &&
(E->hasCopyableElements() || any_of(Scalars, [](Value *Scalar) {
@@ -24663,6 +24729,24 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
case Instruction::Xor: {
setInsertPointAfterBundle(E);
+ // The lanes with the absorbing constant use undef as the other operand.
+ // Freeze the operand, if it may be poison, to keep these lanes equal to
+ // the constant.
+ auto FreezeUndefPlaceholders = [&](Value *Op, unsigned OpIdx) {
+ if (!E->hasCopyableElements() ||
+ none_of(zip(E->Scalars, E->getOperand(OpIdx)),
+ [&](const auto &P) {
+ auto [Scalar, OpScalar] = P;
+ return E->isCopyableElement(Scalar) &&
+ !isa<UndefValue>(Scalar) &&
+ isa<UndefValue>(OpScalar) &&
+ !isa<PoisonValue>(OpScalar);
+ }) ||
+ isGuaranteedNotToBePoison(Op, AC))
+ return Op;
+ return Builder.CreateFreeze(Op);
+ };
+
if (E->hasReassocScalars()) {
// Vectorize operand columns, then combine pairwise in a balanced
// tree; negated columns (flattened subtracts) are subtracted from
@@ -24713,7 +24797,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
continue;
const bool Negated = E->isReassocNegatedOp(Idx);
(Negated ? NegOps : Ops)
- .push_back(GetCastOperand(Idx, vectorizeOperand(E, Idx)));
+ .push_back(GetCastOperand(
+ Idx, FreezeUndefPlaceholders(vectorizeOperand(E, Idx), Idx)));
(Negated ? NegScalarOps : ScalarOps)
.emplace_back(E->getOperand(Idx).begin(),
E->getOperand(Idx).end());
@@ -24870,6 +24955,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
}
}
+ LHS = FreezeUndefPlaceholders(LHS, 0);
+ RHS = FreezeUndefPlaceholders(RHS, 1);
if (LHS->getType() != VecTy || RHS->getType() != VecTy) {
assert((It != MinBWs.end() || getOperandEntry(E, 0)->isGather() ||
getOperandEntry(E, 1)->isGather() ||
@@ -24931,7 +25018,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
FixedVectorType *StridedLoadTy = nullptr;
Value *PO = LI->getPointerOperand();
if (E->State == TreeEntry::Vectorize) {
- NewLI = Builder.CreateAlignedLoad(VecTy, PO, LI->getAlign());
+ const uint64_t Shift = getVectorLoadShift(E->Scalars);
+ if (Shift)
+ PO = Builder.CreatePtrAdd(
+ PO, ConstantInt::getSigned(DL->getIndexType(PO->getType()),
+ -static_cast<int64_t>(Shift)));
+ NewLI = Builder.CreateAlignedLoad(
+ VecTy, PO, commonAlignment(LI->getAlign(), Shift));
} else if (E->State == TreeEntry::CompressVectorize) {
auto [CompressMask, LoadVecTy, InterleaveFactor, IsMasked] =
CompressEntryToData.at(E);
@@ -25048,13 +25141,20 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
NewLI = Builder.CreateMaskedGather(VecTy, VecPtr, CommonAlignment);
}
+ // The wide load also reads the memory of the undef lanes, so the metadata
+ // of the scalar loads does not apply.
Value *V = (E->State == TreeEntry::CompressVectorize ||
- E->State == TreeEntry::BlendedLoadVectorize)
+ E->State == TreeEntry::BlendedLoadVectorize ||
+ E->hasCopyableElements())
? NewLI
: PropagateIRFlags(NewLI);
if (StridedLoadTy != VecTy)
V = Builder.CreateBitOrPointerCast(V, VecTy);
+ // The memory of the undef lanes may hold poison, but the lanes must not
+ // be poison.
+ if (E->hasCopyableElements())
+ V = Builder.CreateFreeze(V);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
index 8279c277a1f04..3cffe3c7d3eee 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
@@ -25,6 +25,7 @@
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/Support/InstructionCost.h"
@@ -363,6 +364,59 @@ bool isMaskedLoadCompress(
InterleaveFactor, CompressMask, LoadVecTy);
}
+bool canWidenLoadsOverUndefLanes(ArrayRef<Value *> VL, const DataLayout &DL,
+ ScalarEvolution &SE, AssumptionCache &AC,
+ const DominatorTree &DT,
+ const TargetLibraryInfo &TLI) {
+ if (count_if(VL, IsaPred<LoadInst>) < 2)
+ return false;
+ const auto *FirstIt = find_if(VL, IsaPred<LoadInst>);
+ const unsigned FirstLane = std::distance(VL.begin(), FirstIt);
+ auto *FirstLI = cast<LoadInst>(*FirstIt);
+ // The wide load may read the memory with a different tag.
+ if (FirstLI->getFunction()->hasFnAttribute(Attribute::SanitizeMemTag))
+ return false;
+ Type *ScalarTy = FirstLI->getType();
+ Value *Ptr0 = FirstLI->getPointerOperand();
+ LoadInst *LastLI = FirstLI;
+ for (auto [Lane, V] : enumerate(VL)) {
+ if (isa<UndefValue>(V) && !isa<PoisonValue>(V))
+ continue;
+ auto *LI = dyn_cast<LoadInst>(V);
+ if (!LI || !LI->isSimple() || mustSuppressSpeculation(*LI))
+ return false;
+ std::optional<int64_t> Dist =
+ getPointersDiff(ScalarTy, Ptr0, ScalarTy, LI->getPointerOperand(), DL,
+ SE, /*StrictCheck=*/true);
+ if (!Dist || *Dist != static_cast<int64_t>(Lane) - FirstLane)
+ return false;
+ if (LastLI->comesBefore(LI))
+ LastLI = LI;
+ }
+ // The wide load starts before the first load, so check the range as the one
+ // starting at the underlying object.
+ const uint64_t EltSize = DL.getTypeAllocSize(ScalarTy).getFixedValue();
+ const unsigned BitWidth = DL.getIndexTypeSizeInBits(Ptr0->getType());
+ APInt Offset(BitWidth, 0);
+ const Value *Base = Ptr0->stripAndAccumulateConstantOffsets(
+ DL, Offset, /*AllowNonInbounds=*/true);
+ bool Overflow = false;
+ APInt Start = Offset.ssub_ov(APInt(BitWidth, FirstLane * EltSize), Overflow);
+ if (Overflow || Start.isNegative())
+ return false;
+ APInt End = Start.sadd_ov(APInt(BitWidth, VL.size() * EltSize), Overflow);
+ return !Overflow &&
+ isDereferenceableAndAlignedPointer(
+ Base, Align(1), End, SimplifyQuery(DL, &TLI, &DT, &AC, LastLI));
+}
+
+uint64_t getVectorLoadShift(ArrayRef<Value *> Scalars) {
+ const auto *It = find_if(Scalars, IsaPred<LoadInst>);
+ auto *LI = cast<LoadInst>(*It);
+ return std::distance(Scalars.begin(), It) *
+ LI->getDataLayout().getTypeAllocSize(LI->getType()).getFixedValue();
+}
+
/// Checks if the stores \p VL with pointers \p PointerOps can be lowered as a
/// single masked store. On success \p StoreVecTy is the widened store type and
/// \p ReuseShuffleIndices is the expand mask that places each stored value at
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
index d1df27e1fb6c3..52a6c73ae542a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
@@ -19,6 +19,8 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/Alignment.h"
+#include <cstdint>
+
namespace llvm {
class AssumptionCache;
class BasicBlock;
@@ -80,6 +82,20 @@ bool isMaskedLoadCompress(
const TargetTransformInfo::TargetCostKind CostKind,
const function_ref<bool(Value *)> AreAllUsersVectorized, bool ReVec);
+/// Checks if the simple loads of \p VL, mixed with the non-poison undef lanes,
+/// can be replaced by a single load of the whole \p VL width. The loads must be
+/// consecutive and match their lanes, and the whole accessed range must be safe
+/// to load unconditionally.
+bool canWidenLoadsOverUndefLanes(ArrayRef<Value *> VL, const DataLayout &DL,
+ ScalarEvolution &SE, AssumptionCache &AC,
+ const DominatorTree &DT,
+ const TargetLibraryInfo &TLI);
+
+/// Returns the distance in bytes from the start of the vector load to the
+/// first load of \p Scalars: the leading undef lanes are covered by the vector
+/// load.
+uint64_t getVectorLoadShift(ArrayRef<Value *> Scalars);
+
/// Checks if the stores \p VL with pointers \p PointerOps can be lowered as a
/// single masked store. On success \p StoreVecTy is the widened store type and
/// \p ReuseShuffleIndices is the expand mask that places each stored value at
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 563f0067b05e9..b87e639774cc5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -45,6 +45,21 @@ bool isBinOpIdentityConstant(const Value *V, unsigned Opcode) {
return CI && ConstantExpr::getBinOpIdentity(Opcode, CI->getType()) == CI;
}
+bool isUndefTolerantBinOp(const Value *V) {
+ const BinaryOperator *BO;
+ return match(V, m_BinOp(BO)) && BO->getType()->isIntOrIntVectorTy() &&
+ !BO->isShift() && !BO->isIntDivRem();
+}
+
+const LoadInst *getLoadOfUndefTolerantBinOp(const Value *V) {
+ if (!isUndefTolerantBinOp(V))
+ return nullptr;
+ for (const Value *Op : cast<BinaryOperator>(V)->operands())
+ if (const auto *LI = dyn_cast<LoadInst>(Op))
+ return LI;
+ return nullptr;
+}
+
unsigned getReassocCombineOpcode(unsigned Opcode) {
switch (Opcode) {
case Instruction::Sub:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index 5d6508dedfad5..34e440be8b69b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -38,6 +38,7 @@ class Constant;
class DataLayout;
class Instruction;
class IRBuilderBase;
+class LoadInst;
class TargetLibraryInfo;
class Type;
class Value;
@@ -59,6 +60,14 @@ bool isConstant(Value *V);
/// returns for FAdd/FMul, whose identity fast-math may break anyway.
bool isBinOpIdentityConstant(const Value *V, unsigned Opcode);
+/// \returns True if \p V is an integer binary operator, which gives an undef
+/// result for the undef operands: neither a shift nor a division.
+bool isUndefTolerantBinOp(const Value *V);
+
+/// \returns the load operand of the undef tolerant binary operator \p V, if
+/// any.
+const LoadInst *getLoadOfUndefTolerantBinOp(const Value *V);
+
/// \returns the opcode of the combines emitted for a reassociated node:
/// subtract chains regroup their positive and negative operand columns with
/// plain adds.
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
index 02f4afd1a46d9..a7e6d5b98175a 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
@@ -9,12 +9,9 @@ define void @foo() {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[ADD277:%.*]] = add nsw i32 undef, undef
; CHECK-NEXT: store i32 [[ADD277]], ptr getelementptr inbounds ([4 x [4 x i32]], ptr @bar, i64 0, i64 3, i64 1), align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr getelementptr inbounds ([4 x [4 x i32]], ptr @bar, i64 0, i64 3, i64 0), align 4
; CHECK-NEXT: [[ARRAYIDX372:%.*]] = getelementptr inbounds [4 x [4 x i32]], ptr @dct_luma, i64 0, i64 3, i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr getelementptr inbounds ([4 x [4 x i32]], ptr @bar, i64 0, i64 3, i64 2), align 4
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 poison, i32 undef, i32 poison, i32 poison>, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([4 x [4 x i32]], ptr @bar, i64 0, i64 3, i64 0), align 4
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> <i32 undef, i32 poison, i32 undef, i32 undef>, i32 [[ADD277]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = ashr <4 x i32> [[TMP5]], splat (i32 6)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll b/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll
index 1d7933334c77d..44284124db9a8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll
@@ -11,16 +11,10 @@ define void @mul_by_index(i32 noundef %x) {
; AVX2-LABEL: define void @mul_by_index(
; AVX2-SAME: i32 noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; AVX2-NEXT: [[ENTRY:.*:]]
-; AVX2-NEXT: [[TMP0:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; AVX2-NEXT: [[TMP7:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; AVX2-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; AVX2-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[TMP0]], i64 1
-; AVX2-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; AVX2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX2-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr @in, align 4
+; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> zeroinitializer
; AVX2-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
; AVX2-NEXT: [[TMP5:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[TMP4]]
; AVX2-NEXT: store <8 x i32> [[TMP5]], ptr @out, align 16
@@ -29,20 +23,15 @@ define void @mul_by_index(i32 noundef %x) {
; SSE4-LABEL: define void @mul_by_index(
; SSE4-SAME: i32 noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; SSE4-NEXT: [[ENTRY:.*:]]
-; SSE4-NEXT: [[TMP0:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; SSE4-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>, i32 [[TMP0]], i64 1
-; SSE4-NEXT: [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE4-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE4-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; SSE4-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; SSE4-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
; SSE4-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
; SSE4-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP4]]
; SSE4-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; SSE4-NEXT: [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; SSE4-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; SSE4-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <4 x i32> zeroinitializer
-; SSE4-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP6]], [[TMP11]]
+; SSE4-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP6]], [[TMP3]]
; SSE4-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP7]], <i32 4, i32 5, i32 6, i32 7>
; SSE4-NEXT: store <4 x i32> [[TMP8]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 16), align 16
; SSE4-NEXT: ret void
@@ -85,17 +74,11 @@ define void @mul_by_index_nsw(i32 noundef %x) {
; AVX2-LABEL: define void @mul_by_index_nsw(
; AVX2-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; AVX2-NEXT: [[ENTRY:.*:]]
-; AVX2-NEXT: [[TMP0:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; AVX2-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; AVX2-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; AVX2-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[TMP0]], i64 1
-; AVX2-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; AVX2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; AVX2-NEXT: [[TMP9:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; AVX2-NEXT: [[TMP4:%.*]] = add nsw <8 x i32> [[TMP7]], [[TMP9]]
+; AVX2-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr @in, align 4
+; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
; AVX2-NEXT: [[TMP5:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[TMP4]]
; AVX2-NEXT: store <8 x i32> [[TMP5]], ptr @out, align 16
; AVX2-NEXT: ret void
@@ -103,19 +86,14 @@ define void @mul_by_index_nsw(i32 noundef %x) {
; SSE4-LABEL: define void @mul_by_index_nsw(
; SSE4-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; SSE4-NEXT: [[ENTRY:.*:]]
-; SSE4-NEXT: [[TMP0:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; SSE4-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>, i32 [[TMP0]], i64 1
-; SSE4-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE4-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE4-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; SSE4-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
-; SSE4-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[TMP11]], [[TMP13]]
+; SSE4-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; SSE4-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; SSE4-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
; SSE4-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP4]]
; SSE4-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; SSE4-NEXT: [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; SSE4-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <4 x i32> zeroinitializer
; SSE4-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP3]]
; SSE4-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP7]], <i32 4, i32 5, i32 6, i32 7>
; SSE4-NEXT: store <4 x i32> [[TMP8]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 16), align 16
@@ -158,16 +136,10 @@ define void @and_with_zero_lane(i32 noundef %x) {
; AVX2-LABEL: define void @and_with_zero_lane(
; AVX2-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; AVX2-NEXT: [[ENTRY:.*:]]
-; AVX2-NEXT: [[TMP0:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; AVX2-NEXT: [[TMP7:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; AVX2-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; AVX2-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> <i32 -1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[TMP0]], i64 1
-; AVX2-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; AVX2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX2-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr @in, align 4
+; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> zeroinitializer
; AVX2-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
; AVX2-NEXT: [[TMP5:%.*]] = and <8 x i32> <i32 0, i32 15, i32 255, i32 4095, i32 65535, i32 1048575, i32 16777215, i32 268435455>, [[TMP4]]
; AVX2-NEXT: store <8 x i32> [[TMP5]], ptr @out, align 16
@@ -176,20 +148,15 @@ define void @and_with_zero_lane(i32 noundef %x) {
; SSE4-LABEL: define void @and_with_zero_lane(
; SSE4-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; SSE4-NEXT: [[ENTRY:.*:]]
-; SSE4-NEXT: [[TMP0:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; SSE4-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 -1, i32 poison, i32 poison, i32 poison>, i32 [[TMP0]], i64 1
-; SSE4-NEXT: [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE4-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE4-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; SSE4-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; SSE4-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
; SSE4-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
; SSE4-NEXT: [[TMP5:%.*]] = and <4 x i32> <i32 0, i32 15, i32 255, i32 4095>, [[TMP4]]
; SSE4-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; SSE4-NEXT: [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; SSE4-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; SSE4-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <4 x i32> zeroinitializer
-; SSE4-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP6]], [[TMP11]]
+; SSE4-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP6]], [[TMP3]]
; SSE4-NEXT: [[TMP8:%.*]] = and <4 x i32> [[TMP7]], <i32 65535, i32 1048575, i32 16777215, i32 268435455>
; SSE4-NEXT: store <4 x i32> [[TMP8]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 16), align 16
; SSE4-NEXT: ret void
@@ -233,16 +200,11 @@ define void @mul_by_index_poison_x(i32 %x) {
; AVX2-LABEL: define void @mul_by_index_poison_x(
; AVX2-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
; AVX2-NEXT: [[ENTRY:.*:]]
-; AVX2-NEXT: [[L1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; AVX2-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; AVX2-NEXT: [[TMP8:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[L1]], i64 1
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP9:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; AVX2-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX2-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr @in, align 4
+; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+; AVX2-NEXT: [[TMP4:%.*]] = freeze <8 x i32> [[TMP3]]
; AVX2-NEXT: [[TMP5:%.*]] = add <8 x i32> [[TMP1]], [[TMP4]]
; AVX2-NEXT: [[TMP6:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[TMP5]]
; AVX2-NEXT: store <8 x i32> [[TMP6]], ptr @out, align 16
@@ -251,19 +213,15 @@ define void @mul_by_index_poison_x(i32 %x) {
; SSE4-LABEL: define void @mul_by_index_poison_x(
; SSE4-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
; SSE4-NEXT: [[ENTRY:.*:]]
-; SSE4-NEXT: [[L1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; SSE4-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 8
-; SSE4-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>, i32 [[L1]], i64 1
-; SSE4-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE4-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE4-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[X]], i64 1
-; SSE4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; SSE4-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; SSE4-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; SSE4-NEXT: [[TMP4:%.*]] = freeze <4 x i32> [[TMP3]]
; SSE4-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
; SSE4-NEXT: [[TMP6:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP5]]
; SSE4-NEXT: store <4 x i32> [[TMP6]], ptr @out, align 16
; SSE4-NEXT: [[TMP7:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
-; SSE4-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <4 x i32> zeroinitializer
; SSE4-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], [[TMP3]]
; SSE4-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP8]], <i32 4, i32 5, i32 6, i32 7>
; SSE4-NEXT: store <4 x i32> [[TMP9]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 16), align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll b/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll
index b77ae7ee40cd2..fb7f96e10dc4d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll
@@ -12,19 +12,13 @@ define void @absorbed_first(i32 noundef %x) {
; CHECK-LABEL: define void @absorbed_first(
; CHECK-SAME: i32 noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: store i32 0, ptr @out, align 16
-; CHECK-NEXT: [[L1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; CHECK-NEXT: [[A1:%.*]] = add i32 [[L1]], [[X]]
-; CHECK-NEXT: [[M1:%.*]] = mul i32 [[A1]], 5
-; CHECK-NEXT: store i32 [[M1]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 4), align 4
-; CHECK-NEXT: [[L2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 4
-; CHECK-NEXT: [[A2:%.*]] = add i32 [[L2]], [[X]]
-; CHECK-NEXT: [[M2:%.*]] = mul i32 [[A2]], 7
-; CHECK-NEXT: store i32 [[M2]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 8), align 4
-; CHECK-NEXT: [[L3:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 12), align 4
-; CHECK-NEXT: [[A3:%.*]] = add i32 [[L3]], [[X]]
-; CHECK-NEXT: [[M3:%.*]] = mul i32 [[A3]], 9
-; CHECK-NEXT: store i32 [[M3]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 12), align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 5, i32 7, i32 9>, [[TMP4]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
entry:
@@ -48,19 +42,13 @@ define void @absorbed_middle(i32 noundef %x) {
; CHECK-LABEL: define void @absorbed_middle(
; CHECK-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[L0:%.*]] = load i32, ptr @in, align 4
-; CHECK-NEXT: [[A0:%.*]] = add i32 [[L0]], [[X]]
-; CHECK-NEXT: [[M0:%.*]] = mul i32 [[A0]], 3
-; CHECK-NEXT: store i32 [[M0]], ptr @out, align 16
-; CHECK-NEXT: [[L1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; CHECK-NEXT: [[A1:%.*]] = add i32 [[L1]], [[X]]
-; CHECK-NEXT: [[M1:%.*]] = mul i32 [[A1]], 5
-; CHECK-NEXT: store i32 [[M1]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 4), align 4
-; CHECK-NEXT: store i32 0, ptr getelementptr inbounds nuw (i8, ptr @out, i64 8), align 4
-; CHECK-NEXT: [[L3:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 12), align 4
-; CHECK-NEXT: [[A3:%.*]] = add i32 [[L3]], [[X]]
-; CHECK-NEXT: [[M3:%.*]] = mul i32 [[A3]], 9
-; CHECK-NEXT: store i32 [[M3]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 12), align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 3, i32 5, i32 0, i32 9>, [[TMP4]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
entry:
@@ -84,19 +72,13 @@ define void @absorbed_last(i32 noundef %x) {
; CHECK-LABEL: define void @absorbed_last(
; CHECK-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[L0:%.*]] = load i32, ptr @in, align 4
-; CHECK-NEXT: [[A0:%.*]] = add i32 [[L0]], [[X]]
-; CHECK-NEXT: [[M0:%.*]] = mul i32 [[A0]], 3
-; CHECK-NEXT: store i32 [[M0]], ptr @out, align 16
-; CHECK-NEXT: [[L1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; CHECK-NEXT: [[A1:%.*]] = add i32 [[L1]], [[X]]
-; CHECK-NEXT: [[M1:%.*]] = mul i32 [[A1]], 5
-; CHECK-NEXT: store i32 [[M1]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 4), align 4
-; CHECK-NEXT: [[L2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 4
-; CHECK-NEXT: [[A2:%.*]] = add i32 [[L2]], [[X]]
-; CHECK-NEXT: [[M2:%.*]] = mul i32 [[A2]], 7
-; CHECK-NEXT: store i32 [[M2]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 8), align 4
-; CHECK-NEXT: store i32 0, ptr getelementptr inbounds nuw (i8, ptr @out, i64 12), align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 3, i32 5, i32 7, i32 0>, [[TMP4]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
entry:
@@ -122,20 +104,14 @@ define void @store_to_absorbed_lane(i32 noundef %x) {
; CHECK-LABEL: define void @store_to_absorbed_lane(
; CHECK-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: store i32 0, ptr @out, align 16
-; CHECK-NEXT: [[L1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
; CHECK-NEXT: store i32 7, ptr @in, align 16
-; CHECK-NEXT: [[A1:%.*]] = add i32 [[L1]], [[X]]
-; CHECK-NEXT: [[M1:%.*]] = mul i32 [[A1]], 5
-; CHECK-NEXT: store i32 [[M1]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 4), align 4
-; CHECK-NEXT: [[L2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 8), align 4
-; CHECK-NEXT: [[A2:%.*]] = add i32 [[L2]], [[X]]
-; CHECK-NEXT: [[M2:%.*]] = mul i32 [[A2]], 7
-; CHECK-NEXT: store i32 [[M2]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 8), align 4
-; CHECK-NEXT: [[L3:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 12), align 4
-; CHECK-NEXT: [[A3:%.*]] = add i32 [[L3]], [[X]]
-; CHECK-NEXT: [[M3:%.*]] = mul i32 [[A3]], 9
-; CHECK-NEXT: store i32 [[M3]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 12), align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 5, i32 7, i32 9>, [[TMP4]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
entry:
@@ -406,17 +382,13 @@ define void @or_disjoint_absorbed_first(i32 noundef %x) {
; CHECK-LABEL: define void @or_disjoint_absorbed_first(
; CHECK-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: store i32 -1, ptr @out, align 16
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 4), align 4
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = or disjoint <2 x i32> [[TMP3]], <i32 256, i32 512>
-; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 4), align 4
-; CHECK-NEXT: [[L3:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @in, i64 12), align 4
-; CHECK-NEXT: [[A3:%.*]] = add i32 [[L3]], [[X]]
-; CHECK-NEXT: [[M3:%.*]] = or disjoint i32 [[A3]], 1024
-; CHECK-NEXT: store i32 [[M3]], ptr getelementptr inbounds nuw (i8, ptr @out, i64 12), align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> <i32 -1, i32 256, i32 512, i32 1024>, [[TMP4]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
entry:
>From a25ed4c90921ed2bbff5a02ac2b33ae3dbe7ed68 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 4 Oct 2026 05:07:34 -0700
Subject: [PATCH 2/2] Rebase, improvements
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 68 ++++++++++---------
.../SLPVectorizer/SLPMemoryUtils.cpp | 2 +-
.../Vectorize/SLPVectorizer/SLPMemoryUtils.h | 2 +-
.../X86/absorbing-copyable-lane.ll | 32 +++++----
.../X86/wide-load-absorbed-lane.ll | 25 ++++---
5 files changed, 73 insertions(+), 56 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3dce3946841cb..53c42b2888e62 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6304,7 +6304,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
if (DL->getTypeSizeInBits(ScalarTy) != DL->getTypeAllocSizeInBits(ScalarTy))
return LoadsState::Gather;
- // The undef lanes are covered by a single wide load of the consecutive loads.
+ // The undef and poison lanes are covered by a single wide load of the
+ // consecutive loads.
if (any_of(VL, IsaPred<UndefValue>))
return canWidenLoadsOverUndefLanes(VL, *DL, *SE, *AC, *DT, *TLI)
? LoadsState::Vectorize
@@ -10277,6 +10278,10 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
// duplicates) set can be scatter/compress-vectorized. Prefer the unique
// loads (pack + reshuffle) when possible, otherwise use the originals.
if (S && S.getOpcode() == Instruction::Load) {
+ // The poison lanes are covered by a single wide load, nothing to pack.
+ if (any_of(VL, IsaPred<PoisonValue>) &&
+ GetLoadsState(VL) == BoUpSLP::LoadsState::Vectorize)
+ return std::make_pair(true, true);
bool UniquesVectorized =
CheckLoads(UniqueValues, /*IncludeGather=*/false);
if (UniquesVectorized || CheckLoads(VL, /*IncludeGather=*/false)) {
@@ -10764,11 +10769,11 @@ class InstructionsCompatibilityAnalysis {
/// instruction and its actual operands should be returned, or it is a
/// copyable element and its should be represented as idempotent instruction.
/// \p SelfOp selects the op(V, V) modeling for a copyable lane instead of
- /// op(V, identity). \p UndefPlaceholder selects op(V, undef) for the
+ /// op(V, identity). \p PoisonPlaceholder selects op(V, poison) for the
/// absorbing constant V.
SmallVector<Value *> getOperands(const InstructionsState &S, Value *V,
bool SelfOp,
- bool UndefPlaceholder = false) const {
+ bool PoisonPlaceholder = false) const {
if (SelfOp)
return {V, V};
if (isa<PoisonValue>(V)) {
@@ -10811,19 +10816,20 @@ class InstructionsCompatibilityAnalysis {
cast<CastInst>(MainOp)->getSrcTy())};
assert(isSupportedMainOp(MainOp) && "Unsupported opcode");
Type *Ty = V->getType();
- if (UndefPlaceholder && V == ConstantExpr::getBinOpAbsorber(MainOpcode, Ty))
- return {V, UndefValue::get(Ty)};
+ if (PoisonPlaceholder &&
+ V == ConstantExpr::getBinOpAbsorber(MainOpcode, Ty))
+ return {V, PoisonValue::get(Ty)};
if (isa<UndefValue>(V) && isUndefTolerantBinOp(MainOp))
return {V, V};
return {V, selectBestIdempotentValue()};
}
- /// Checks if the lanes with the absorbing constant may use undef as the other
- /// operand: the other operands of the remaining lanes are the binary
- /// operators of the different loads, whose undef lane is covered by the wide
+ /// Checks if the lanes with the absorbing constant may use poison as the
+ /// other operand: the other operands of the remaining lanes are the binary
+ /// operators of the different loads, whose poison lane is covered by the wide
/// load.
- bool canUseUndefPlaceholders(const InstructionsState &S,
- ArrayRef<Value *> VL) const {
+ bool canUsePoisonPlaceholders(const InstructionsState &S,
+ ArrayRef<Value *> VL) const {
Constant *Absorber =
ConstantExpr::getBinOpAbsorber(MainOpcode, VL.front()->getType());
if (!Absorber || !is_contained(VL, Absorber))
@@ -11612,11 +11618,11 @@ class InstructionsCompatibilityAnalysis {
Operands.assign(NumMainOpOperands,
BoUpSLP::ValueList(VL.size(), nullptr));
SmallPtrSet<const Value *, 4> SelfOpLanes = findSelfOpLanes(S, VL, R);
- const bool UndefPlaceholders = canUseUndefPlaceholders(S, VL);
+ const bool PoisonPlaceholders = canUsePoisonPlaceholders(S, VL);
// Populate operands for every lane.
for (auto [Idx, V] : enumerate(VL)) {
SmallVector<Value *> OperandsForValue =
- getOperands(S, V, SelfOpLanes.contains(V), UndefPlaceholders);
+ getOperands(S, V, SelfOpLanes.contains(V), PoisonPlaceholders);
for (auto [OperandIdx, Operand] : enumerate(OperandsForValue))
Operands[OperandIdx][Idx] = Operand;
}
@@ -14798,7 +14804,8 @@ void BoUpSLP::transformNodes() {
case Instruction::Load: {
// No need to reorder masked gather loads, just reorder the scalar
// operands.
- if (E.State != TreeEntry::Vectorize || E.hasCopyableElements())
+ if (E.State != TreeEntry::Vectorize ||
+ any_of(E.Scalars, IsaPred<UndefValue>))
break;
Type *ScalarTy = E.getMainOp()->getType();
auto *VecTy =
@@ -24063,10 +24070,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if (!AllNoInfs)
I->setHasNoInfs(false);
}
- // Copyable lanes modeled as op(V, V) or op(V, undef) do not preserve the
- // disjoint flag: or disjoint(V, V) is poison unless V is zero, and the
- // undef operand may be any value. Matched by value, so operand reordering
- // does not affect the detection.
+ // Copyable lanes modeled as op(V, V) or with an undef or poison operand do
+ // not preserve the disjoint flag: or disjoint(V, V) is poison unless V is
+ // zero, and such an operand may be any value. Matched by value, so operand
+ // reordering does not affect the detection.
auto *PDI = dyn_cast<PossiblyDisjointInst>(I);
if (E->hasCopyableElements() && PDI &&
any_of(enumerate(E->Scalars), [&](const auto &P) {
@@ -24729,18 +24736,17 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
case Instruction::Xor: {
setInsertPointAfterBundle(E);
- // The lanes with the absorbing constant use undef as the other operand.
+ // The lanes with the absorbing constant use poison as the other operand.
// Freeze the operand, if it may be poison, to keep these lanes equal to
// the constant.
- auto FreezeUndefPlaceholders = [&](Value *Op, unsigned OpIdx) {
+ auto FreezePlaceholders = [&](Value *Op, unsigned OpIdx) {
if (!E->hasCopyableElements() ||
none_of(zip(E->Scalars, E->getOperand(OpIdx)),
[&](const auto &P) {
auto [Scalar, OpScalar] = P;
return E->isCopyableElement(Scalar) &&
!isa<UndefValue>(Scalar) &&
- isa<UndefValue>(OpScalar) &&
- !isa<PoisonValue>(OpScalar);
+ isa<UndefValue>(OpScalar);
}) ||
isGuaranteedNotToBePoison(Op, AC))
return Op;
@@ -24798,7 +24804,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
const bool Negated = E->isReassocNegatedOp(Idx);
(Negated ? NegOps : Ops)
.push_back(GetCastOperand(
- Idx, FreezeUndefPlaceholders(vectorizeOperand(E, Idx), Idx)));
+ Idx, FreezePlaceholders(vectorizeOperand(E, Idx), Idx)));
(Negated ? NegScalarOps : ScalarOps)
.emplace_back(E->getOperand(Idx).begin(),
E->getOperand(Idx).end());
@@ -24955,8 +24961,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
}
}
- LHS = FreezeUndefPlaceholders(LHS, 0);
- RHS = FreezeUndefPlaceholders(RHS, 1);
+ LHS = FreezePlaceholders(LHS, 0);
+ RHS = FreezePlaceholders(RHS, 1);
if (LHS->getType() != VecTy || RHS->getType() != VecTy) {
assert((It != MinBWs.end() || getOperandEntry(E, 0)->isGather() ||
getOperandEntry(E, 1)->isGather() ||
@@ -25141,19 +25147,19 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
NewLI = Builder.CreateMaskedGather(VecTy, VecPtr, CommonAlignment);
}
- // The wide load also reads the memory of the undef lanes, so the metadata
- // of the scalar loads does not apply.
+ // The wide load also reads the memory of the undef and poison lanes, so
+ // the metadata of the scalar loads does not apply.
+ const bool HasUndefLanes = any_of(E->Scalars, IsaPred<UndefValue>);
Value *V = (E->State == TreeEntry::CompressVectorize ||
- E->State == TreeEntry::BlendedLoadVectorize ||
- E->hasCopyableElements())
+ E->State == TreeEntry::BlendedLoadVectorize || HasUndefLanes)
? NewLI
: PropagateIRFlags(NewLI);
if (StridedLoadTy != VecTy)
V = Builder.CreateBitOrPointerCast(V, VecTy);
- // The memory of the undef lanes may hold poison, but the lanes must not
- // be poison.
- if (E->hasCopyableElements())
+ // The memory of the undef and poison lanes may hold poison, but these
+ // lanes may be used, so they must not be poison.
+ if (HasUndefLanes)
V = Builder.CreateFreeze(V);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
index 3cffe3c7d3eee..7a02b20943b0b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
@@ -380,7 +380,7 @@ bool canWidenLoadsOverUndefLanes(ArrayRef<Value *> VL, const DataLayout &DL,
Value *Ptr0 = FirstLI->getPointerOperand();
LoadInst *LastLI = FirstLI;
for (auto [Lane, V] : enumerate(VL)) {
- if (isa<UndefValue>(V) && !isa<PoisonValue>(V))
+ if (isa<UndefValue>(V))
continue;
auto *LI = dyn_cast<LoadInst>(V);
if (!LI || !LI->isSimple() || mustSuppressSpeculation(*LI))
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
index 52a6c73ae542a..c3b86caf0bae7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
@@ -82,7 +82,7 @@ bool isMaskedLoadCompress(
const TargetTransformInfo::TargetCostKind CostKind,
const function_ref<bool(Value *)> AreAllUsersVectorized, bool ReVec);
-/// Checks if the simple loads of \p VL, mixed with the non-poison undef lanes,
+/// Checks if the simple loads of \p VL, mixed with the undef and poison lanes,
/// can be replaced by a single load of the whole \p VL width. The loads must be
/// consecutive and match their lanes, and the whole accessed range must be safe
/// to load unconditionally.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll b/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll
index 44284124db9a8..9fdefbbebe004 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/absorbing-copyable-lane.ll
@@ -14,9 +14,10 @@ define void @mul_by_index(i32 noundef %x) {
; AVX2-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr @in, align 4
; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
; AVX2-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
-; AVX2-NEXT: [[TMP5:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[TMP4]]
+; AVX2-NEXT: [[TMP6:%.*]] = freeze <8 x i32> [[TMP4]]
+; AVX2-NEXT: [[TMP5:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[TMP6]]
; AVX2-NEXT: store <8 x i32> [[TMP5]], ptr @out, align 16
; AVX2-NEXT: ret void
;
@@ -28,7 +29,8 @@ define void @mul_by_index(i32 noundef %x) {
; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
; SSE4-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SSE4-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP4]]
+; SSE4-NEXT: [[TMP9:%.*]] = freeze <4 x i32> [[TMP4]]
+; SSE4-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP9]]
; SSE4-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; SSE4-NEXT: [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
; SSE4-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP6]], [[TMP3]]
@@ -77,8 +79,9 @@ define void @mul_by_index_nsw(i32 noundef %x) {
; AVX2-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr @in, align 4
; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> zeroinitializer
-; AVX2-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+; AVX2-NEXT: [[TMP6:%.*]] = add nsw <8 x i32> [[TMP1]], [[TMP3]]
+; AVX2-NEXT: [[TMP4:%.*]] = freeze <8 x i32> [[TMP6]]
; AVX2-NEXT: [[TMP5:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[TMP4]]
; AVX2-NEXT: store <8 x i32> [[TMP5]], ptr @out, align 16
; AVX2-NEXT: ret void
@@ -90,7 +93,8 @@ define void @mul_by_index_nsw(i32 noundef %x) {
; SSE4-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
-; SSE4-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; SSE4-NEXT: [[TMP9:%.*]] = add nsw <4 x i32> [[TMP1]], [[TMP3]]
+; SSE4-NEXT: [[TMP4:%.*]] = freeze <4 x i32> [[TMP9]]
; SSE4-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP4]]
; SSE4-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; SSE4-NEXT: [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
@@ -139,9 +143,10 @@ define void @and_with_zero_lane(i32 noundef %x) {
; AVX2-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr @in, align 4
; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
; AVX2-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
-; AVX2-NEXT: [[TMP5:%.*]] = and <8 x i32> <i32 0, i32 15, i32 255, i32 4095, i32 65535, i32 1048575, i32 16777215, i32 268435455>, [[TMP4]]
+; AVX2-NEXT: [[TMP6:%.*]] = freeze <8 x i32> [[TMP4]]
+; AVX2-NEXT: [[TMP5:%.*]] = and <8 x i32> <i32 0, i32 15, i32 255, i32 4095, i32 65535, i32 1048575, i32 16777215, i32 268435455>, [[TMP6]]
; AVX2-NEXT: store <8 x i32> [[TMP5]], ptr @out, align 16
; AVX2-NEXT: ret void
;
@@ -153,7 +158,8 @@ define void @and_with_zero_lane(i32 noundef %x) {
; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
; SSE4-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SSE4-NEXT: [[TMP5:%.*]] = and <4 x i32> <i32 0, i32 15, i32 255, i32 4095>, [[TMP4]]
+; SSE4-NEXT: [[TMP9:%.*]] = freeze <4 x i32> [[TMP4]]
+; SSE4-NEXT: [[TMP5:%.*]] = and <4 x i32> <i32 0, i32 15, i32 255, i32 4095>, [[TMP9]]
; SSE4-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; SSE4-NEXT: [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
; SSE4-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP6]], [[TMP3]]
@@ -204,8 +210,8 @@ define void @mul_by_index_poison_x(i32 %x) {
; AVX2-NEXT: [[TMP1:%.*]] = freeze <8 x i32> [[TMP0]]
; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
-; AVX2-NEXT: [[TMP4:%.*]] = freeze <8 x i32> [[TMP3]]
-; AVX2-NEXT: [[TMP5:%.*]] = add <8 x i32> [[TMP1]], [[TMP4]]
+; AVX2-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
+; AVX2-NEXT: [[TMP5:%.*]] = freeze <8 x i32> [[TMP4]]
; AVX2-NEXT: [[TMP6:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[TMP5]]
; AVX2-NEXT: store <8 x i32> [[TMP6]], ptr @out, align 16
; AVX2-NEXT: ret void
@@ -217,8 +223,8 @@ define void @mul_by_index_poison_x(i32 %x) {
; SSE4-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
; SSE4-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; SSE4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
-; SSE4-NEXT: [[TMP4:%.*]] = freeze <4 x i32> [[TMP3]]
-; SSE4-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
+; SSE4-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; SSE4-NEXT: [[TMP5:%.*]] = freeze <4 x i32> [[TMP4]]
; SSE4-NEXT: [[TMP6:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP5]]
; SSE4-NEXT: store <4 x i32> [[TMP6]], ptr @out, align 16
; SSE4-NEXT: [[TMP7:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @in, i64 16), align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll b/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll
index fb7f96e10dc4d..2ab10a86b807f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/wide-load-absorbed-lane.ll
@@ -15,9 +15,10 @@ define void @absorbed_first(i32 noundef %x) {
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 5, i32 7, i32 9>, [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i32> [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 5, i32 7, i32 9>, [[TMP6]]
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
@@ -45,9 +46,10 @@ define void @absorbed_middle(i32 noundef %x) {
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 0>
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 3, i32 5, i32 0, i32 9>, [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i32> [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 3, i32 5, i32 0, i32 9>, [[TMP6]]
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
@@ -75,9 +77,10 @@ define void @absorbed_last(i32 noundef %x) {
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 poison>
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 3, i32 5, i32 7, i32 0>, [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i32> [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 3, i32 5, i32 7, i32 0>, [[TMP6]]
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
@@ -108,9 +111,10 @@ define void @store_to_absorbed_lane(i32 noundef %x) {
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 5, i32 7, i32 9>, [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i32> [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> <i32 0, i32 5, i32 7, i32 9>, [[TMP6]]
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
@@ -385,9 +389,10 @@ define void @or_disjoint_absorbed_first(i32 noundef %x) {
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @in, align 4
; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> <i32 -1, i32 256, i32 512, i32 1024>, [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i32> [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> <i32 -1, i32 256, i32 512, i32 1024>, [[TMP6]]
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr @out, align 16
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list