[llvm] [SandboxVec][LoadStoreVec] Vectorize load seed chains (PR #215944)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 07:06:03 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/215944
>From 09c40b8418902f64d59af84f04fbbf3c25aa9c0a Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Thu, 13 Aug 2026 01:21:54 -0400
Subject: [PATCH] [SandboxVectorizer] Dispatch LoadStoreVec::runOnRegion on
seed kind
---
.../SandboxVectorizer/Passes/LoadStoreVec.h | 53 +++-
.../Vectorize/SandboxVectorizer/VecUtils.h | 3 +-
.../SandboxVectorizer/Passes/LoadStoreVec.cpp | 233 +++++++++++-------
.../Vectorize/SandboxVectorizer/VecUtils.cpp | 17 +-
.../Passes/LoadStoreVec/load_store_vec.ll | 70 +++---
.../LoadStoreVec/load_store_vec_load_seeds.ll | 105 ++++++++
.../SandboxVectorizer/VecUtilsTest.cpp | 10 +-
7 files changed, 340 insertions(+), 151 deletions(-)
create mode 100644 llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_load_seeds.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h
index d4d29ec6da6ef..0289c4fb18889 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h
@@ -17,6 +17,10 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/SandboxIR/Pass.h"
#include "llvm/Support/InstructionCost.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+#include <optional>
namespace llvm {
@@ -24,9 +28,10 @@ class DataLayout;
namespace sandboxir {
+class Context;
+class Function;
class Value;
class Instruction;
-class Scheduler;
class Type;
class LLVM_ABI LoadStoreVec final : public RegionPass {
@@ -35,33 +40,55 @@ class LLVM_ABI LoadStoreVec final : public RegionPass {
Region *SavedRgn = nullptr;
/// The region's cost at the time of saveIR().
InstructionCost CostBefore = 0;
+ const Analyses *A = nullptr;
+ Context *Ctx = nullptr;
+ std::optional<Scheduler> Sched;
+ VecUtils::DeadInstructionMorgue DeadInstrMorgue;
+
+ /// Initializes \c A, \c Ctx, and \c Sched for \p F using \p AnalysesRef.
+ void initialize(Function &F, const Analyses &AnalysesRef);
/// Saves the IR along with the current cost of \p Rgn, so that
/// acceptOrRevert() can tell whether vectorizing was profitable.
void saveIR(Region &Rgn);
/// Accepts the transaction saved by saveIR() if vectorizing was profitable,
- /// reverts it otherwise. \Returns true if the transaction was accepted.
+ /// reverts it otherwise. \returns true if the transaction was accepted.
bool acceptOrRevert();
/// Checks legality of vectorization and \returns the vector type on success,
/// nullopt otherwise.
- std::optional<Type *> canVectorize(ArrayRef<Instruction *> Bndl,
- Scheduler &Sched);
-
- void tryEraseDeadInstrs(ArrayRef<Instruction *> Stores,
- ArrayRef<Value *> Operands);
-
- /// Tries to vectorize the load/store/constant ops chain \p Bndl
- /// into a single vector store. \Returns whether it succeeded.
- bool vectorizeStores(ArrayRef<Instruction *> Bndl, Region &Rgn,
- Scheduler &Sched, const Analyses &A);
+ std::optional<Type *> canVectorize(ArrayRef<Instruction *> Bndl);
+
+ /// Builds a single vector load out of the load operands in \p Loads.
+ /// \returns the new load, or nullptr if \p Loads are not a vectorizable
+ /// load chain.
+ LoadInst *createVectorLoad(ArrayRef<Instruction *> Loads);
+
+ /// Builds a ConstantVector from per-lane constant store operands in \p
+ /// Constants. Aggregates and sequential constants contribute their elements
+ /// in order; aggregate-zero and splat vector constants are expanded to one
+ /// element per lane. \returns the packed ConstantVector.
+ Value *createConstantVector(ArrayRef<Value *> Constants);
+
+ /// Tries to vectorize the store bundle \p Bndl into a single vector store.
+ /// Load value-operands are vectorized via vectorizeLoads(); constant
+ /// operands are packed into a ConstantVector. \returns whether it succeeded.
+ bool vectorizeStores(ArrayRef<Instruction *> Bndl, Region &Rgn);
+
+ /// Tries to vectorize the load chain \p Loads into a single vector load.
+ /// If \p ManageTransaction is true, also unpacks for remaining uses, erases
+ /// dead original loads, and save/accept-or-reverts. If false, only emits
+ /// the vector load; the caller owns the IR transaction.
+ /// \returns the new vector load, or nullptr on failure.
+ LoadInst *vectorizeLoads(ArrayRef<Instruction *> Loads, Region &Rgn,
+ bool ManageTransaction = true);
public:
LoadStoreVec(StringRef AuxArg) : RegionPass("load-store-vec") {
assert(AuxArg.empty() && "This pass ignores aux arg!");
}
- bool runOnRegion(Region &Rgn, const Analyses &A) final;
+ bool runOnRegion(Region &Rgn, const Analyses &AnalysesRef) final;
};
} // namespace sandboxir
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 80e266d1c3022..57f077a433040 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -12,6 +12,7 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
+#include "llvm/ADT/DenseSet.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/SandboxIR/Type.h"
@@ -413,7 +414,7 @@ class VecUtils {
/// For load/store bundles, also record non-first-lane pointer operands;
/// the first lane's pointer is skipped because the vector load/store
/// reuses it. Erased later by \c tryEraseDeadInstrs().
- void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
+ template <typename T> void collectPotentiallyDeadInstrs(ArrayRef<T *> Bndl);
/// Erase candidates recorded by \c collectPotentiallyDeadInstrs() that
/// now have no uses, then clear the candidate set.
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp
index 08cd8a48efb87..6580f8b557f00 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
#include "llvm/Support/CommandLine.h"
@@ -25,8 +26,13 @@ namespace sandboxir {
#define DEBUG_PREFIX_LOCAL DEBUG_PREFIX "LoadStoreVec: "
-std::optional<Type *> LoadStoreVec::canVectorize(ArrayRef<Instruction *> Bndl,
- Scheduler &Sched) {
+void LoadStoreVec::initialize(Function &F, const Analyses &AnalysesRef) {
+ A = &AnalysesRef;
+ Ctx = &F.getContext();
+ Sched.emplace(A->getAA(), *Ctx, SchedDirection::BottomUp);
+}
+
+std::optional<Type *> LoadStoreVec::canVectorize(ArrayRef<Instruction *> Bndl) {
// Check if in the same BB.
if (LegalityAnalysis::differentBlock(Bndl))
return std::nullopt;
@@ -36,34 +42,12 @@ std::optional<Type *> LoadStoreVec::canVectorize(ArrayRef<Instruction *> Bndl,
return std::nullopt;
// Check scheduling.
- if (!Sched.trySchedule(Bndl))
+ if (!Sched->trySchedule(Bndl))
return std::nullopt;
return VecUtils::getCombinedVectorTypeFor(Bndl, *DL);
}
-void LoadStoreVec::tryEraseDeadInstrs(ArrayRef<Instruction *> Stores,
- ArrayRef<Value *> Operands) {
- SmallPtrSet<Instruction *, 8> DeadCandidates;
- for (auto *SI : Stores) {
- if (auto *PtrI =
- dyn_cast<Instruction>(cast<StoreInst>(SI)->getPointerOperand()))
- DeadCandidates.insert(PtrI);
- SI->eraseFromParent();
- }
- for (auto *Op : Operands) {
- if (auto *LI = dyn_cast<LoadInst>(Op)) {
- if (auto *PtrI =
- dyn_cast<Instruction>(cast<LoadInst>(LI)->getPointerOperand()))
- DeadCandidates.insert(PtrI);
- cast<LoadInst>(LI)->eraseFromParent();
- }
- }
- for (auto *PtrI : DeadCandidates)
- if (!PtrI->hasNUsesOrMore(1))
- PtrI->eraseFromParent();
-}
-
void LoadStoreVec::saveIR(Region &Rgn) {
SavedRgn = &Rgn;
const auto &SB = cast<RegionWithScore>(Rgn).getScoreboard();
@@ -72,7 +56,7 @@ void LoadStoreVec::saveIR(Region &Rgn) {
}
bool LoadStoreVec::acceptOrRevert() {
- auto &Ctx = SavedRgn->getContext();
+ auto &SavedCtx = SavedRgn->getContext();
const auto &SB = cast<RegionWithScore>(*SavedRgn).getScoreboard();
InstructionCost CostAfter = SB.getAfterCost() - SB.getBeforeCost();
InstructionCost CostGain = CostAfter - CostBefore;
@@ -81,26 +65,77 @@ bool LoadStoreVec::acceptOrRevert() {
<< ")\n");
if (CostGain > CostThreshold) {
LLVM_DEBUG(dbgs() << DEBUG_PREFIX_LOCAL << "Not profitable, reverting.\n");
- Ctx.revert();
+ SavedCtx.revert();
return false;
}
LLVM_DEBUG(dbgs() << DEBUG_PREFIX_LOCAL << "Profitable accepting.\n");
- Ctx.accept();
+ SavedCtx.accept();
return true;
}
-bool LoadStoreVec::vectorizeStores(ArrayRef<Instruction *> Bndl, Region &Rgn,
- Scheduler &Sched, const Analyses &A) {
- Function &F = *Bndl[0]->getParent()->getParent();
- auto &Ctx = F.getContext();
+LoadInst *LoadStoreVec::createVectorLoad(ArrayRef<Instruction *> Loads) {
+ if (!VecUtils::areConsecutive<LoadInst, Instruction>(
+ Loads, A->getScalarEvolution(), *DL))
+ return nullptr;
+ if (!canVectorize(Loads))
+ return nullptr;
+
+ Type *Ty = VecUtils::getCombinedVectorTypeFor(Loads, *DL);
+ Value *LdPtr = cast<LoadInst>(Loads[0])->getPointerOperand();
+ // TODO: Compute alignment.
+ Align LdAlign(1);
+ auto LdWhereIt = std::next(VecUtils::getLowest(Loads)->getIterator());
+ return LoadInst::create(Ty, LdPtr, LdAlign, LdWhereIt, *Ctx, "VecIinitL");
+}
+
+Value *LoadStoreVec::createConstantVector(ArrayRef<Value *> Operands) {
+ SmallVector<Constant *, 8> Constants;
+ Constants.reserve(Operands.size());
+ for (Value *Op : Operands) {
+ auto *COp = cast<Constant>(Op);
+ if (auto *AggrCOp = dyn_cast<ConstantAggregate>(COp)) {
+ // If the operand is a constant aggregate, then append all its elements.
+ for (Value *Elm : AggrCOp->operands())
+ Constants.push_back(cast<Constant>(Elm));
+ } else if (auto *SeqCOp = dyn_cast<ConstantDataSequential>(COp)) {
+ for (auto ElmIdx : seq<unsigned>(SeqCOp->getNumElements()))
+ Constants.push_back(SeqCOp->getElementAsConstant(ElmIdx));
+ } else if (auto *Zero = dyn_cast<ConstantAggregateZero>(COp)) {
+ auto *ZeroElm = Zero->getSequentialElement();
+ for ([[maybe_unused]] auto Cnt :
+ seq<unsigned>(Zero->getElementCount().getFixedValue()))
+ Constants.push_back(ZeroElm);
+ } else if (isa<ConstantInt>(COp) && isa<VectorType>(COp->getType())) {
+ auto *Elm = ConstantInt::get(*Ctx, cast<ConstantInt>(COp)->getValue());
+ for ([[maybe_unused]] auto Cnt :
+ seq<unsigned>(cast<VectorType>(COp->getType())
+ ->getElementCount()
+ .getFixedValue()))
+ Constants.push_back(Elm);
+ } else if (isa<ConstantFP>(COp) && isa<VectorType>(COp->getType())) {
+ auto *Elm = ConstantFP::get(cast<ConstantFP>(COp)->getValue(), *Ctx);
+ for ([[maybe_unused]] auto Cnt :
+ seq<unsigned>(cast<VectorType>(COp->getType())
+ ->getElementCount()
+ .getFixedValue()))
+ Constants.push_back(Elm);
+ } else {
+ Constants.push_back(COp);
+ }
+ }
+ return ConstantVector::get(Constants);
+}
+
+bool LoadStoreVec::vectorizeStores(ArrayRef<Instruction *> Bndl, Region &Rgn) {
+ assert(all_of(Bndl, [](Instruction *I) { return isa<StoreInst>(I); }) &&
+ "vectorizeStores expects a store bundle!");
if (!VecUtils::areConsecutive<StoreInst, Instruction>(
- Bndl, A.getScalarEvolution(), *DL))
+ Bndl, A->getScalarEvolution(), *DL))
return false;
- if (!canVectorize(Bndl, Sched))
+ if (!canVectorize(Bndl))
return false;
- SmallVector<Value *, 4> Operands;
- Operands.reserve(Bndl.size());
+ SmallVector<Value *> Operands;
for (auto *I : Bndl) {
auto *Op = cast<StoreInst>(I)->getValueOperand();
Operands.push_back(Op);
@@ -131,66 +166,17 @@ bool LoadStoreVec::vectorizeStores(ArrayRef<Instruction *> Bndl, Region &Rgn,
Value *VecOp = nullptr;
if (AllLoads) {
- // TODO: Try to avoid the extra copy to an instruction vector.
SmallVector<Instruction *, 8> Loads;
Loads.reserve(Operands.size());
for (Value *Op : Operands)
Loads.push_back(cast<Instruction>(Op));
-
- bool Consecutive = VecUtils::areConsecutive<LoadInst, Instruction>(
- Loads, A.getScalarEvolution(), *DL);
- if (!Consecutive) {
- Ctx.accept();
+ VecOp = vectorizeLoads(Loads, Rgn, /*ManageTransaction=*/false);
+ if (VecOp == nullptr) {
+ Ctx->accept();
return false;
}
- if (!canVectorize(Loads, Sched)) {
- Ctx.accept();
- return false;
- }
-
- // Generate vector load.
- Type *Ty = VecUtils::getCombinedVectorTypeFor(Bndl, *DL);
- Value *LdPtr = cast<LoadInst>(Loads[0])->getPointerOperand();
- // TODO: Compute alignment.
- Align LdAlign(1);
- auto LdWhereIt = std::next(VecUtils::getLowest(Loads)->getIterator());
- VecOp = LoadInst::create(Ty, LdPtr, LdAlign, LdWhereIt, Ctx, "VecIinitL");
} else if (AllConstants) {
- SmallVector<Constant *, 8> Constants;
- Constants.reserve(Operands.size());
- for (Value *Op : Operands) {
- auto *COp = cast<Constant>(Op);
- if (auto *AggrCOp = dyn_cast<ConstantAggregate>(COp)) {
- // If the operand is a constant aggregate, then append all its elements.
- for (Value *Elm : AggrCOp->operands())
- Constants.push_back(cast<Constant>(Elm));
- } else if (auto *SeqCOp = dyn_cast<ConstantDataSequential>(COp)) {
- for (auto ElmIdx : seq<unsigned>(SeqCOp->getNumElements()))
- Constants.push_back(SeqCOp->getElementAsConstant(ElmIdx));
- } else if (auto *Zero = dyn_cast<ConstantAggregateZero>(COp)) {
- auto *ZeroElm = Zero->getSequentialElement();
- for ([[maybe_unused]] auto Cnt :
- seq<unsigned>(Zero->getElementCount().getFixedValue()))
- Constants.push_back(ZeroElm);
- } else if (isa<ConstantInt>(COp) && isa<VectorType>(COp->getType())) {
- auto *Elm = ConstantInt::get(Ctx, cast<ConstantInt>(COp)->getValue());
- for ([[maybe_unused]] auto Cnt :
- seq<unsigned>(cast<VectorType>(COp->getType())
- ->getElementCount()
- .getFixedValue()))
- Constants.push_back(Elm);
- } else if (isa<ConstantFP>(COp) && isa<VectorType>(COp->getType())) {
- auto *Elm = ConstantFP::get(cast<ConstantFP>(COp)->getValue(), Ctx);
- for ([[maybe_unused]] auto Cnt :
- seq<unsigned>(cast<VectorType>(COp->getType())
- ->getElementCount()
- .getFixedValue()))
- Constants.push_back(Elm);
- } else {
- Constants.push_back(COp);
- }
- }
- VecOp = ConstantVector::get(Constants);
+ VecOp = createConstantVector(Operands);
}
// Generate vector store.
@@ -198,21 +184,80 @@ bool LoadStoreVec::vectorizeStores(ArrayRef<Instruction *> Bndl, Region &Rgn,
// TODO: Compute alignment.
Align StAlign(1);
auto StWhereIt = std::next(VecUtils::getLowest(Bndl)->getIterator());
- StoreInst::create(VecOp, StPtr, StAlign, StWhereIt, Ctx);
+ StoreInst::create(VecOp, StPtr, StAlign, StWhereIt, *Ctx);
- tryEraseDeadInstrs(Bndl, Operands);
+ DeadInstrMorgue.collectPotentiallyDeadInstrs(Bndl);
+ if (AllLoads)
+ DeadInstrMorgue.collectPotentiallyDeadInstrs<Value>(Operands);
+ DeadInstrMorgue.tryEraseDeadInstrs();
return acceptOrRevert();
}
-bool LoadStoreVec::runOnRegion(Region &Rgn, const Analyses &A) {
+LoadInst *LoadStoreVec::vectorizeLoads(ArrayRef<Instruction *> Bndl,
+ Region &Rgn, bool ManageTransaction) {
+ if (ManageTransaction)
+ saveIR(Rgn);
+
+ auto *VecLoad = createVectorLoad(Bndl);
+ if (VecLoad == nullptr) {
+ if (ManageTransaction)
+ Ctx->accept();
+ return nullptr;
+ }
+
+ if (!ManageTransaction)
+ return VecLoad;
+
+ // TODO: Support mixed-type top-level load chains.
+ Type *VecElemTy = cast<FixedVectorType>(VecLoad->getType())->getElementType();
+ if (!all_of(Bndl, [VecElemTy](Instruction *I) {
+ return VecUtils::getElementType(I->getType()) == VecElemTy;
+ })) {
+ Ctx->revert();
+ return nullptr;
+ }
+
+ BasicBlock::iterator WhereIt = std::next(VecLoad->getIterator());
+ for (auto [Lane, OrigV] : VecUtils::enumerateLanes(Bndl)) {
+ auto *OrigLoad = cast<LoadInst>(OrigV);
+ if (OrigLoad->hasNUses(0))
+ continue;
+ Value *Unpacked =
+ VecUtils::unpack(VecLoad, OrigLoad->getType(), Lane, WhereIt);
+ OrigLoad->replaceAllUsesWith(Unpacked);
+ }
+
+ DeadInstrMorgue.collectPotentiallyDeadInstrs(Bndl);
+ DeadInstrMorgue.tryEraseDeadInstrs();
+
+ if (!acceptOrRevert())
+ return nullptr;
+ return VecLoad;
+}
+
+bool LoadStoreVec::runOnRegion(Region &Rgn, const Analyses &AnalysesRef) {
SmallVector<Instruction *, 8> Bndl(Rgn.getAux().begin(), Rgn.getAux().end());
if (Bndl.size() < 2)
return false;
Function &F = *Bndl[0]->getParent()->getParent();
DL = &F.getParent()->getDataLayout();
- Scheduler Sched(A.getAA(), F.getContext(), SchedDirection::BottomUp);
- return vectorizeStores(Bndl, Rgn, Sched, A);
+ initialize(F, AnalysesRef);
+
+ // SeedCollection only ever gives us a homogeneous seed slice: stores and
+ // loads are collected in separate passes over the BB, never mixed into one
+ // Aux (see SeedCollection::runOnFunction).
+ bool IsStoreKind = isa<StoreInst>(Bndl[0]);
+ assert(all_of(Bndl,
+ [&](Instruction *I) {
+ return isa<StoreInst>(I) == IsStoreKind;
+ }) &&
+ "Expected a homogeneous seed slice!");
+
+ bool Changed = IsStoreKind ? vectorizeStores(Bndl, Rgn)
+ : vectorizeLoads(Bndl, Rgn) != nullptr;
+ Sched.reset();
+ return Changed;
}
} // namespace sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
index abf3d79966d8f..6611ae5a110ca 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
@@ -8,6 +8,7 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/SandboxIR/Instruction.h"
@@ -104,22 +105,23 @@ unsigned VecUtils::getFloorPowerOf2(unsigned Num) {
return Num & ~Mask;
}
+template <typename T>
void VecUtils::DeadInstructionMorgue::collectPotentiallyDeadInstrs(
- ArrayRef<Value *> Bndl) {
- for (Value *V : Bndl)
+ ArrayRef<T *> Bndl) {
+ for (T *V : Bndl)
DeadInstrCandidates.insert(cast<Instruction>(V));
// Also collect the GEPs of vectorized loads and stores.
auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
switch (Opcode) {
case Instruction::Opcode::Load: {
- for (Value *V : drop_begin(Bndl))
+ for (T *V : drop_begin(Bndl))
if (auto *Ptr =
dyn_cast<Instruction>(cast<LoadInst>(V)->getPointerOperand()))
DeadInstrCandidates.insert(Ptr);
break;
}
case Instruction::Opcode::Store: {
- for (Value *V : drop_begin(Bndl))
+ for (T *V : drop_begin(Bndl))
if (auto *Ptr =
dyn_cast<Instruction>(cast<StoreInst>(V)->getPointerOperand()))
DeadInstrCandidates.insert(Ptr);
@@ -130,6 +132,13 @@ void VecUtils::DeadInstructionMorgue::collectPotentiallyDeadInstrs(
}
}
+template void
+ VecUtils::DeadInstructionMorgue::collectPotentiallyDeadInstrs<Value>(
+ ArrayRef<Value *>);
+template void
+ VecUtils::DeadInstructionMorgue::collectPotentiallyDeadInstrs<Instruction>(
+ ArrayRef<Instruction *>);
+
void VecUtils::DeadInstructionMorgue::tryEraseDeadInstrs() {
DenseMap<BasicBlock *, SmallVector<Instruction *>> SortedDeadInstrCandidates;
// The dead instrs could span BBs, so we need to collect and sort them per BB.
diff --git a/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec.ll b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec.ll
index 4708871dee4fd..06f0f060eb2a4 100644
--- a/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec.ll
@@ -6,8 +6,8 @@ define void @load_store_vec_basic(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_basic(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
-; CHECK-NEXT: store <4 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META0]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: store <4 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META1]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -26,8 +26,8 @@ define void @load_store_vec_non_pow2(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_non_pow2(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <3 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META1:![0-9]+]]
-; CHECK-NEXT: store <3 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META1]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <3 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: store <3 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META2]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -43,8 +43,8 @@ define void @load_store_vec_vectorize_vectors(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_vectorize_vectors(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META2:![0-9]+]]
-; CHECK-NEXT: store <4 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: store <4 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META3]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -60,8 +60,8 @@ define void @load_store_vec_vectorize_vectors_diff_types(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_vectorize_vectors_diff_types(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <8 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META3:![0-9]+]]
-; CHECK-NEXT: store <8 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META3]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <8 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META4:![0-9]+]]
+; CHECK-NEXT: store <8 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META4]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -84,8 +84,8 @@ define void @load_store_vec_gap(ptr %ptr) {
; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 2
; CHECK-NEXT: [[LD0:%.*]] = load i8, ptr [[PTR0]], align 1
; CHECK-NEXT: [[LD1:%.*]] = load i16, ptr [[PTR1]], align 2
-; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META4:![0-9]+]]
-; CHECK-NEXT: store i16 [[LD1]], ptr [[PTR1]], align 2, !sandboxvec [[META4]]
+; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META5:![0-9]+]]
+; CHECK-NEXT: store i16 [[LD1]], ptr [[PTR1]], align 2, !sandboxvec [[META5]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -104,8 +104,8 @@ define void @load_store_vec_loads_not_consecutive(ptr %ptr) {
; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 2
; CHECK-NEXT: [[LD0:%.*]] = load i8, ptr [[PTR1]], align 1
; CHECK-NEXT: [[LD1:%.*]] = load i16, ptr [[PTR0]], align 2
-; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META5:![0-9]+]]
-; CHECK-NEXT: store i16 [[LD1]], ptr [[PTR1]], align 2, !sandboxvec [[META5]]
+; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META6:![0-9]+]]
+; CHECK-NEXT: store i16 [[LD1]], ptr [[PTR1]], align 2, !sandboxvec [[META6]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -122,8 +122,8 @@ define void @load_store_vec_same_types(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_same_types(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META6:![0-9]+]]
-; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META6]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META7:![0-9]+]]
+; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META7]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -140,8 +140,8 @@ define void @load_store_vec_mixed_int_float(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_mixed_int_float(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META7:![0-9]+]]
-; CHECK-NEXT: store <2 x i32> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META7]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META8:![0-9]+]]
+; CHECK-NEXT: store <2 x i32> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META8]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i32, ptr %ptr, i32 0
@@ -157,8 +157,8 @@ define void @load_store_vec_mixed_int_float_vectors(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_mixed_int_float_vectors(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <3 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META8:![0-9]+]]
-; CHECK-NEXT: store <3 x i32> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <3 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META9:![0-9]+]]
+; CHECK-NEXT: store <3 x i32> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META9]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i32, ptr %ptr, i32 0
@@ -181,8 +181,8 @@ define ptr @load_store_vec_dont_cross_bbs(ptr %ptr) {
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[PTR1]], align 1
; CHECK-NEXT: br label %[[BB:.*]]
; CHECK: [[BB]]:
-; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META9:![0-9]+]]
-; CHECK-NEXT: store i8 [[LD1]], ptr [[PTR1]], align 1, !sandboxvec [[META9]]
+; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META10:![0-9]+]]
+; CHECK-NEXT: store i8 [[LD1]], ptr [[PTR1]], align 1, !sandboxvec [[META10]]
; CHECK-NEXT: ret ptr [[PTR1]]
;
entry:
@@ -204,8 +204,8 @@ define void @load_store_vec_cleanup_geps(ptr %ptrA, ptr %ptrB) {
; CHECK-SAME: ptr [[PTRA:%.*]], ptr [[PTRB:%.*]]) {
; CHECK-NEXT: [[PTRA0:%.*]] = getelementptr i8, ptr [[PTRA]], i32 0
; CHECK-NEXT: [[PTRB0:%.*]] = getelementptr i8, ptr [[PTRB]], i32 0
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTRA0]], align 1, !sandboxvec [[META10:![0-9]+]]
-; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTRB0]], align 1, !sandboxvec [[META10]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTRA0]], align 1, !sandboxvec [[META11:![0-9]+]]
+; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTRB0]], align 1, !sandboxvec [[META11]]
; CHECK-NEXT: ret void
;
%ptrA0 = getelementptr i8, ptr %ptrA, i32 0
@@ -225,8 +225,8 @@ define ptr @load_store_vec_cleanup_gep_with_external_use(ptr %ptr) {
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 1
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META11:![0-9]+]]
-; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META11]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META12:![0-9]+]]
+; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META12]]
; CHECK-NEXT: ret ptr [[PTR1]]
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -246,9 +246,9 @@ define void @load_store_vec_schedule_stores_and_loads(ptr noalias %ptrA, ptr noa
; CHECK-NEXT: [[PTRB0:%.*]] = getelementptr i8, ptr [[PTRB]], i64 0
; CHECK-NEXT: [[PTRB1:%.*]] = getelementptr i8, ptr [[PTRB]], i64 1
; CHECK-NEXT: [[OTHER:%.*]] = load i8, ptr [[PTRB1]], align 1
-; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTRA0]], align 1, !sandboxvec [[META12:![0-9]+]]
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTRA0]], align 1, !sandboxvec [[META13:![0-9]+]]
; CHECK-NEXT: store i8 0, ptr [[PTRA0]], align 1
-; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTRB0]], align 1, !sandboxvec [[META12]]
+; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTRB0]], align 1, !sandboxvec [[META13]]
; CHECK-NEXT: ret void
;
%ptrA0 = getelementptr i8, ptr %ptrA, i64 0
@@ -276,9 +276,9 @@ define void @load_store_vec_external_uses(ptr %ptr) {
; CHECK-NEXT: [[LD0:%.*]] = load i8, ptr [[PTR0]], align 1
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[PTR1]], align 1
; CHECK-NEXT: [[LD2:%.*]] = load i16, ptr [[PTR2]], align 2
-; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META13:![0-9]+]]
-; CHECK-NEXT: store i8 [[LD1]], ptr [[PTR1]], align 1, !sandboxvec [[META13]]
-; CHECK-NEXT: store i16 [[LD2]], ptr [[PTR2]], align 2, !sandboxvec [[META14:![0-9]+]]
+; CHECK-NEXT: store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META14:![0-9]+]]
+; CHECK-NEXT: store i8 [[LD1]], ptr [[PTR1]], align 1, !sandboxvec [[META14]]
+; CHECK-NEXT: store i16 [[LD2]], ptr [[PTR2]], align 2, !sandboxvec [[META15:![0-9]+]]
; CHECK-NEXT: [[EXT_USER:%.*]] = zext i8 [[LD1]] to i132
; CHECK-NEXT: ret void
;
@@ -301,7 +301,7 @@ define void @load_store_vec_constants(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: store <3 x i8> <i8 42, i8 43, i8 44>, ptr [[PTR0]], align 1, !sandboxvec [[META15:![0-9]+]]
+; CHECK-NEXT: store <3 x i8> <i8 42, i8 43, i8 44>, ptr [[PTR0]], align 1, !sandboxvec [[META16:![0-9]+]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -318,7 +318,7 @@ define void @load_store_vec_constants_CDS(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants_CDS(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: store <4 x i8> <i8 0, i8 1, i8 2, i8 3>, ptr [[PTR0]], align 1, !sandboxvec [[META16:![0-9]+]]
+; CHECK-NEXT: store <4 x i8> <i8 0, i8 1, i8 2, i8 3>, ptr [[PTR0]], align 1, !sandboxvec [[META17:![0-9]+]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -335,7 +335,7 @@ define void @load_store_vec_constants_CDS_float(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants_CDS_float(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: store <8 x float> <float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, ptr [[PTR0]], align 1, !sandboxvec [[META17:![0-9]+]]
+; CHECK-NEXT: store <8 x float> <float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, ptr [[PTR0]], align 1, !sandboxvec [[META18:![0-9]+]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
@@ -352,7 +352,7 @@ define void @load_store_vec_constants_CI_vector(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants_CI_vector(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
-; CHECK-NEXT: store <8 x i8> <i8 0, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 2>, ptr [[PTR0]], align 1, !sandboxvec [[META18:![0-9]+]]
+; CHECK-NEXT: store <8 x i8> <i8 0, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 2>, ptr [[PTR0]], align 1, !sandboxvec [[META19:![0-9]+]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
@@ -369,7 +369,7 @@ define void @load_store_vec_constants_CFP_vector(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants_CFP_vector(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: store <8 x float> <float 1.000000e+00, float 2.000000e+00, float 2.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>, ptr [[PTR0]], align 1, !sandboxvec [[META19:![0-9]+]]
+; CHECK-NEXT: store <8 x float> <float 1.000000e+00, float 2.000000e+00, float 2.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>, ptr [[PTR0]], align 1, !sandboxvec [[META20:![0-9]+]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
@@ -382,7 +382,6 @@ define void @load_store_vec_constants_CFP_vector(ptr %ptr) {
}
;.
-; CHECK: [[META0]] = distinct !{!"sandboxregion"}
; CHECK: [[META1]] = distinct !{!"sandboxregion"}
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
@@ -402,4 +401,5 @@ define void @load_store_vec_constants_CFP_vector(ptr %ptr) {
; CHECK: [[META17]] = distinct !{!"sandboxregion"}
; CHECK: [[META18]] = distinct !{!"sandboxregion"}
; CHECK: [[META19]] = distinct !{!"sandboxregion"}
+; CHECK: [[META20]] = distinct !{!"sandboxregion"}
;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_load_seeds.ll b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_load_seeds.ll
new file mode 100644
index 0000000000000..f08c2c6dbd96b
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_load_seeds.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=sandbox-vectorizer -sbvec-passes="seed-collection(enable-diff-types)<load-store-vec>" -sbvec-collect-seeds=loads,stores -sbvec-vec-reg-bits=1024 %s -S | FileCheck %s
+
+define void @load_store_vec_trivial_load_defuse_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: @load_store_vec_trivial_load_defuse_store(
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i32 0
+; CHECK-NEXT: [[PTRS0:%.*]] = getelementptr i8, ptr [[PTR2:%.*]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META4:![0-9]+]]
+; CHECK-NEXT: store <2 x i8> [[VECIINITL]], ptr [[PTRS0]], align 1, !sandboxvec [[META4]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 1
+ %ld0 = load i8, ptr %ptr0
+ %ld1 = load i8, ptr %ptr1
+ %ptrs0 = getelementptr i8, ptr %ptr2, i32 0
+ %ptrs1 = getelementptr i8, ptr %ptr2, i32 1
+ store i8 %ld0, ptr %ptrs0
+ store i8 %ld1, ptr %ptrs1
+ ret void
+}
+
+define void @vectorize_loads_no_uses(ptr %ptr) {
+; CHECK-LABEL: @vectorize_loads_no_uses(
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 1
+ %ld0 = load i8, ptr %ptr0
+ %ld1 = load i8, ptr %ptr1
+ ret void
+}
+
+define i8 @vectorize_loads_single_use(ptr %ptr) {
+; CHECK-LABEL: @vectorize_loads_single_use(
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i8> [[VECIINITL]], i32 0, !sandboxvec [[META1]]
+; CHECK-NEXT: ret i8 [[UNPACK]]
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 1
+ %ld0 = load i8, ptr %ptr0
+ %ld1 = load i8, ptr %ptr1
+ ret i8 %ld0
+}
+
+define i8 @vectorize_loads_general_uses(ptr %ptr) {
+; CHECK-LABEL: @vectorize_loads_general_uses(
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i8> [[VECIINITL]], i32 0, !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i8> [[VECIINITL]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[SUM:%.*]] = add i8 [[UNPACK]], [[UNPACK1]]
+; CHECK-NEXT: ret i8 [[SUM]]
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 1
+ %ld0 = load i8, ptr %ptr0
+ %ld1 = load i8, ptr %ptr1
+ %sum = add i8 %ld0, %ld1
+ ret i8 %sum
+}
+
+; A vector-typed original load must be replaced with a shuffle rather than a
+; plain extractelement (see VecUtils::unpack()), unlike the scalar loads in
+; the other tests here.
+define <2 x i8> @vectorize_loads_vector_typed(ptr %ptr) {
+; CHECK-LABEL: @vectorize_loads_vector_typed(
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = shufflevector <4 x i8> [[VECIINITL]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>, !sandboxvec [[META3]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = shufflevector <4 x i8> [[VECIINITL]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>, !sandboxvec [[META3]]
+; CHECK-NEXT: [[SUM:%.*]] = add <2 x i8> [[UNPACK]], [[UNPACK1]]
+; CHECK-NEXT: ret <2 x i8> [[SUM]]
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 2
+ %ld0 = load <2 x i8>, ptr %ptr0
+ %ld1 = load <2 x i8>, ptr %ptr1
+ %sum = add <2 x i8> %ld0, %ld1
+ ret <2 x i8> %sum
+}
+
+; TODO: Following should be vectorized
+define i32 @vectorize_loads_mixed_types_not_vectorized(ptr %ptr) {
+; CHECK-LABEL: @vectorize_loads_mixed_types_not_vectorized(
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i32 0
+; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 4
+; CHECK-NEXT: [[LD0:%.*]] = load i32, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[LD1:%.*]] = load i16, ptr [[PTR1]], align 2
+; CHECK-NEXT: [[EXT:%.*]] = zext i16 [[LD1]] to i32
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[LD0]], [[EXT]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 4
+ %ld0 = load i32, ptr %ptr0
+ %ld1 = load i16, ptr %ptr1
+ %ext = zext i16 %ld1 to i32
+ %sum = add i32 %ld0, %ext
+ ret i32 %sum
+}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
index b4f40a1eee201..efa19efbac89d 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -1153,7 +1153,8 @@ define void @scalar(i8 %v) {
auto *Ret = cast<sandboxir::Instruction>(&*It++);
sandboxir::VecUtils::DeadInstructionMorgue Morgue;
- Morgue.collectPotentiallyDeadInstrs({Live, Dead0, Dead1});
+ Morgue.collectPotentiallyDeadInstrs<sandboxir::Instruction>(
+ {Live, Dead0, Dead1});
Morgue.tryEraseDeadInstrs();
// %dead0 and %dead1 had no uses, so they should have been erased. %live is
@@ -1189,7 +1190,7 @@ define void @loadtest(ptr %p) {
auto *Ret = cast<sandboxir::Instruction>(&*It++);
sandboxir::VecUtils::DeadInstructionMorgue Morgue;
- Morgue.collectPotentiallyDeadInstrs({Ld0, Ld1});
+ Morgue.collectPotentiallyDeadInstrs<sandboxir::Instruction>({Ld0, Ld1});
Morgue.tryEraseDeadInstrs();
// %ld0 and %ld1 are collected directly and have no uses, so they are
@@ -1227,7 +1228,7 @@ define void @storetest(ptr %p, float %v) {
auto *Ret = cast<sandboxir::Instruction>(&*It++);
sandboxir::VecUtils::DeadInstructionMorgue Morgue;
- Morgue.collectPotentiallyDeadInstrs({St0, St1});
+ Morgue.collectPotentiallyDeadInstrs<sandboxir::Instruction>({St0, St1});
Morgue.tryEraseDeadInstrs();
// %st0 and %st1 are collected directly (stores are always "used" 0 times)
@@ -1257,7 +1258,8 @@ define void @clears(i8 %v) {
auto *Ret = cast<sandboxir::Instruction>(&*It++);
sandboxir::VecUtils::DeadInstructionMorgue Morgue;
- Morgue.collectPotentiallyDeadInstrs({Dead0});
+ Morgue.collectPotentiallyDeadInstrs<sandboxir::Instruction>(
+ SmallVector<sandboxir::Instruction *, 1>({Dead0}));
Morgue.tryEraseDeadInstrs();
// The candidate set should have been cleared by the first call, so a
// second call must be a safe no-op rather than trying to dereference the
More information about the llvm-commits
mailing list