[llvm] [SandboxVectorizer] Implement topdown vectorizer (PR #205249)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 22:55:59 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/205249
>From f06744bbb411f66b11e9f545a30b25d32265b392 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Sun, 14 Jun 2026 17:01:48 -0400
Subject: [PATCH] [SBVec] Implement topDown/botUp vectorizers in unison
This patch introduces the `top-down-vec` pass to the Sandbox Vectorizer,
adding the ability to traverse use-def chains top-down to discover and
collect vectorization opportunities. Furthermore, this patch unifies
the two vectorizers into a single implementation to minimize code
duplication.
---
.../SandboxVectorizer/Passes/BottomUpVec.h | 77 +--
.../SandboxVectorizer/Passes/BundleVecPass.h | 143 +++++
.../SandboxVectorizer/Passes/TopDownVec.h | 26 +
.../Vectorize/SandboxVectorizer/VecUtils.h | 8 +-
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 2 +-
.../{BottomUpVec.cpp => BundleVecPass.cpp} | 599 +++++++++---------
.../SandboxVectorizer/Passes/PassRegistry.def | 1 +
.../SandboxVectorizerPassBuilder.cpp | 1 +
.../SandboxVectorizer/external_uses.ll | 68 ++
.../test/Transforms/SandboxVectorizer/pack.ll | 77 ++-
.../SandboxVectorizer/topdown_vec.ll | 94 +++
.../llvm/lib/Transforms/Vectorize/BUILD.gn | 2 +-
12 files changed, 721 insertions(+), 377 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
rename llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/{BottomUpVec.cpp => BundleVecPass.cpp} (50%)
create mode 100644 llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index d024bb74ac0c9..08590add639fb 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -12,13 +12,7 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/SandboxIR/Constant.h"
-#include "llvm/SandboxIR/Pass.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h"
namespace llvm::sandboxir {
@@ -32,74 +26,9 @@ namespace llvm::sandboxir {
/// profitable or not. For now profitability is checked at the end of the region
/// pass pipeline by a dedicated pass that accepts or rejects the IR
/// transaction, depending on the cost.
-class LLVM_ABI BottomUpVec final : public RegionPass {
- bool Change = false;
- /// The original instructions that are potentially dead after vectorization.
- DenseSet<Instruction *> DeadInstrCandidates;
- /// Maps scalars to vectors.
- std::unique_ptr<InstrMaps> IMaps;
- /// Counter used for force-stopping the vectorizer after this many
- /// invocations. Used for debugging miscompiles.
- unsigned long BottomUpInvocationCnt = 0;
-
- /// Creates and returns a vector instruction that replaces the instructions in
- /// \p Bndl. \p Operands are the already vectorized operands.
- Value *createVectorInstr(ArrayRef<Value *> Bndl, ArrayRef<Value *> Operands);
- /// Erases all dead instructions from the dead instruction candidates
- /// collected during vectorization.
- void tryEraseDeadInstrs();
- /// Creates a shuffle instruction that shuffles \p VecOp according to \p Mask.
- /// \p UserBB is the block of the user bundle.
- Value *createShuffle(Value *VecOp, const ShuffleMask &Mask,
- BasicBlock *UserBB);
- /// Packs all elements of \p ToPack into a vector and returns that vector. \p
- /// UserBB is the block of the user bundle.
- Value *createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB);
- /// After we create vectors for groups of instructions, the original
- /// instructions are potentially dead and may need to be removed. This
- /// function helps collect these instructions (along with the pointer operands
- /// for loads/stores) so that they can be cleaned up later.
- void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
-
- /// Helper class describing how(if) to vectorize the code.
- class ActionsVector {
- private:
- SmallVector<std::unique_ptr<Action>, 16> Actions;
-
- public:
- auto begin() const { return Actions.begin(); }
- auto end() const { return Actions.end(); }
- void push_back(std::unique_ptr<Action> &&ActPtr) {
- ActPtr->Idx = Actions.size();
- Actions.push_back(std::move(ActPtr));
- }
- void clear() { Actions.clear(); }
-#ifndef NDEBUG
- void print(raw_ostream &OS) const;
- void dump() const;
-#endif // NDEBUG
- };
- ActionsVector Actions;
- /// Helper counter for debugging. It counts the bundles that we attempt to
- /// vectorize in vectorizeRec().
- unsigned DebugBndlCnt = 0;
-
- /// Recursively try to vectorize \p Bndl and its operands. This populates the
- /// `Actions` vector.
- Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
- unsigned Depth, LegalityAnalysis &Legality);
- /// If the values in \p Bndl have external users, then emit unpacks and
- /// connect them to the users. \p Vec is the vectorized form of \p Bndl.
- void emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl, Value *Vec);
- /// Generate vector instructions based on `Actions` and return the last vector
- /// created.
- Value *emitVectors();
- /// Entry point for vectorization starting from \p Seeds.
- bool tryVectorize(ArrayRef<Value *> Seeds, LegalityAnalysis &Legality);
-
+class LLVM_ABI BottomUpVec final : public BundleVecPass {
public:
- BottomUpVec() : RegionPass("bottom-up-vec") {}
- bool runOnRegion(Region &Rgn, const Analyses &A) final;
+ BottomUpVec() : BundleVecPass("bottom-up-vec", VecDirection::BottomUp) {}
};
} // namespace llvm::sandboxir
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h
new file mode 100644
index 0000000000000..63c9657014960
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h
@@ -0,0 +1,143 @@
+//===- BundleVecPass.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Region pass that vectorizes instruction bundles, either top-down or
+// bottom-up.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BUNDLEVECPASS_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BUNDLEVECPASS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+namespace llvm::sandboxir {
+
+enum class VecDirection { TopDown, BottomUp };
+
+/// Vectorizes instruction bundles starting from a seed slice in the region.
+/// The traversal direction is selected via \p VecDirection.
+class LLVM_ABI BundleVecPass : public RegionPass {
+protected:
+ VecDirection Direction;
+ /// Set to true whenever the pass emits vector code in the current region.
+ bool Change = false;
+ /// Scalar instructions that may be dead after vectorization.
+ DenseSet<Instruction *> DeadInstrCandidates;
+ /// Maps scalar values/instructions to their vector counterparts.
+ std::unique_ptr<InstrMaps> IMaps;
+ /// Helper counter for debugging — counts bundles attempted in vectorizeRec.
+ unsigned DebugBndlCnt = 0;
+
+ /// Ordered list of vectorization decisions produced by vectorizeRec.
+ class ActionsVector {
+ SmallVector<std::unique_ptr<Action>, 16> Actions;
+
+ public:
+ auto begin() const { return Actions.begin(); }
+ auto end() const { return Actions.end(); }
+ auto rbegin() const { return Actions.rbegin(); }
+ auto rend() const { return Actions.rend(); }
+ void push_back(std::unique_ptr<Action> &&ActPtr) {
+ ActPtr->Idx = Actions.size();
+ Actions.push_back(std::move(ActPtr));
+ }
+ void clear() { Actions.clear(); }
+#ifndef NDEBUG
+ void print(raw_ostream &OS) const {
+ for (auto [Idx, Action] : enumerate(Actions)) {
+ Action->print(OS);
+ OS << "\n";
+ }
+ }
+ LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
+#endif // NDEBUG
+ };
+ ActionsVector Actions;
+
+ explicit BundleVecPass(StringRef Name, VecDirection Dir)
+ : RegionPass(Name), Direction(Dir) {}
+
+ /// Recursively try to vectorize \p Bndl. For bottom-up vectorization \p
+ /// UserBndl tracks the bundle of users that led to this recursion.
+ Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
+ unsigned Depth, LegalityAnalysis &Legality);
+
+ /// \Returns the operand at \p OpIdx for each instruction in \p Bndl.
+ static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
+ unsigned OpIdx) {
+ SmallVector<Value *, 4> Operands;
+ for (Value *BndlV : Bndl)
+ Operands.push_back(cast<Instruction>(BndlV)->getOperand(OpIdx));
+ return Operands;
+ }
+
+ /// \Returns the BB iterator after the lowest instruction in \p Vals, or the
+ /// top of BB (after any PHIs) if no instruction is found in \p Vals.
+ static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef<Value *> Vals,
+ BasicBlock *BB) {
+ auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB));
+ if (BotI == nullptr)
+ return BB->empty() ? BB->begin()
+ : std::next(VecUtils::getLastPHIOrSelf(&*BB->begin())
+ ->getIterator());
+ return std::next(BotI->getIterator());
+ }
+
+ /// Creates and returns a new vector instruction widening \p Bndl with
+ /// \p Operands as the vector operands.
+ static Value *createVectorInstr(ArrayRef<Value *> Bndl,
+ ArrayRef<Value *> Operands);
+
+ /// Creates a shuffle of \p VecOp according to \p Mask, inserted in \p UserBB.
+ static Value *createShuffle(Value *VecOp, const ShuffleMask &Mask,
+ BasicBlock *UserBB);
+
+ /// Packs all scalars/vectors in \p ToPack into a single vector, inserted in
+ /// \p UserBB.
+ static Value *createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB);
+
+ /// Adds the instructions in \p Bndl (and pointer operands of loads/stores)
+ /// to DeadInstrCandidates for later cleanup.
+ void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
+
+ /// Erases all zero-use instructions from DeadInstrCandidates.
+ /// \p PassName is used as the debug-output prefix.
+ void tryEraseDeadInstrs(StringRef PassName);
+
+ /// For each element of \p Bndl that has external uses (users not yet
+ /// vectorized), emits extract instructions from \p Vec and replaces the
+ /// scalar uses with them.
+ void emitUnpacksForExternalUses(ArrayRef<Value *> Bndl, Value *Vec);
+
+ /// Generate vector instructions based on `Actions` and return the last vector
+ /// created.
+ Value *emitVectors();
+
+ /// \Returns true if vectorization should be skipped due to the stop-at limit.
+ bool shouldSkipVectorization();
+
+private:
+ unsigned long InvocationCnt = 0;
+
+ bool tryVectorize(ArrayRef<Value *> Bndl, LegalityAnalysis &Legality);
+
+public:
+ bool runOnRegion(Region &Rgn, const Analyses &A) override;
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BUNDLEVECPASS_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
new file mode 100644
index 0000000000000..2aebf6aae9e48
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
@@ -0,0 +1,26 @@
+//===- TopDownVec.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A Top-Down Vectorizer pass.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h"
+
+namespace llvm::sandboxir {
+
+class LLVM_ABI TopDownVec final : public BundleVecPass {
+public:
+ TopDownVec() : BundleVecPass("top-down-vec", VecDirection::TopDown) {}
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 99e411a1723b0..57fb884c27699 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -12,6 +12,8 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/SandboxIR/Type.h"
@@ -33,6 +35,10 @@ template <> struct DenseMapInfo<SmallVector<sandboxir::Value *>> {
namespace sandboxir {
+class ShuffleMask;
+class InstrMaps;
+class BundleVecPass;
+
class VecUtils {
public:
/// \Returns the number of elements in \p Ty. That is the number of lanes if a
@@ -379,7 +385,7 @@ class VecUtils {
}
#ifndef NDEBUG
- /// Helper dump function for debugging.
+ /// Print \p Bndl. Dump function for debugging.
LLVM_DUMP_METHOD static void dump(ArrayRef<Value *> Bndl);
LLVM_DUMP_METHOD static void dump(ArrayRef<Instruction *> Bndl);
#endif // NDEBUG
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index d081747b821a4..f823fed93c80f 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -8,7 +8,7 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/InstrMaps.cpp
SandboxVectorizer/Interval.cpp
SandboxVectorizer/Legality.cpp
- SandboxVectorizer/Passes/BottomUpVec.cpp
+ SandboxVectorizer/Passes/BundleVecPass.cpp
SandboxVectorizer/Passes/LoadStoreVec.cpp
SandboxVectorizer/Passes/PackReuse.cpp
SandboxVectorizer/Passes/RegionsFromBBs.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.cpp
similarity index 50%
rename from llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
rename to llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.cpp
index 6bf257fcf8b1d..4a889fe1433fd 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.cpp
@@ -1,4 +1,4 @@
-//===- BottomUpVec.cpp - A bottom-up vectorizer pass ----------------------===//
+//===- BundleVecPass.cpp - Bundle vectorizer pass -------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,27 +6,19 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h"
#include "llvm/SandboxIR/Function.h"
#include "llvm/SandboxIR/Instruction.h"
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
#include "llvm/SandboxIR/Utils.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Debug.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
namespace llvm {
-#ifndef NDEBUG
-static cl::opt<bool>
- AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
- cl::desc("Helps find bugs by verifying the IR whenever we "
- "emit new instructions (*very* expensive)."));
-#endif // NDEBUG
-
static constexpr unsigned long StopAtDisabled =
std::numeric_limits<unsigned long>::max();
+
static cl::opt<unsigned long>
StopAt("sbvec-stop-at", cl::init(StopAtDisabled), cl::Hidden,
cl::desc("Vectorize if the invocation count is < than this. 0 "
@@ -34,229 +26,182 @@ static cl::opt<unsigned long>
static constexpr unsigned long StopBundleDisabled =
std::numeric_limits<unsigned long>::max();
+
static cl::opt<unsigned long>
- StopBundle("sbvec-stop-bndl", cl::init(StopBundleDisabled), cl::Hidden,
- cl::desc("Vectorize up to this many bundles."));
+ BottomUpStopBundle("sbvec-stop-bndl", cl::init(StopBundleDisabled),
+ cl::Hidden,
+ cl::desc("Vectorize up to this many bundles."));
-namespace sandboxir {
+#ifndef NDEBUG
+static cl::opt<bool>
+ AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
+ cl::desc("Helps find bugs by verifying the IR whenever "
+ "we emit new instructions (*very* expensive)."));
+#endif // NDEBUG
+} // namespace llvm
-static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
- unsigned OpIdx) {
- SmallVector<Value *, 4> Operands;
- for (Value *BndlV : Bndl) {
- auto *BndlI = cast<Instruction>(BndlV);
- Operands.push_back(BndlI->getOperand(OpIdx));
- }
- return Operands;
-}
+namespace llvm::sandboxir {
-/// \Returns the BB iterator after the lowest instruction in \p Vals, or the top
-/// of BB if no instruction found in \p Vals.
-static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef<Value *> Vals,
- BasicBlock *BB) {
- auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB));
- if (BotI == nullptr)
- // We are using BB->begin() (or after PHIs) as the fallback insert point.
- return BB->empty()
- ? BB->begin()
- : std::next(
- VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
- return std::next(BotI->getIterator());
-}
+Value *BundleVecPass::createVectorInstr(ArrayRef<Value *> Bndl,
+ ArrayRef<Value *> Operands) {
+ assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
+ "Expect Instructions!");
+ auto &Ctx = Bndl[0]->getContext();
-Value *BottomUpVec::createVectorInstr(ArrayRef<Value *> Bndl,
- ArrayRef<Value *> Operands) {
- auto CreateVectorInstr = [](ArrayRef<Value *> Bndl,
- ArrayRef<Value *> Operands) -> Value * {
- assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
- "Expect Instructions!");
- auto &Ctx = Bndl[0]->getContext();
-
- Type *ScalarTy = VecUtils::getElementType(Utils::getExpectedType(Bndl[0]));
- auto *VecTy = VecUtils::getWideType(ScalarTy, VecUtils::getNumLanes(Bndl));
-
- BasicBlock::iterator WhereIt = getInsertPointAfterInstrs(
- Bndl, cast<Instruction>(Bndl[0])->getParent());
-
- auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
- switch (Opcode) {
- case Instruction::Opcode::ZExt:
- case Instruction::Opcode::SExt:
- case Instruction::Opcode::FPToUI:
- case Instruction::Opcode::FPToSI:
- case Instruction::Opcode::FPExt:
- case Instruction::Opcode::PtrToInt:
- case Instruction::Opcode::IntToPtr:
- case Instruction::Opcode::SIToFP:
- case Instruction::Opcode::UIToFP:
- case Instruction::Opcode::Trunc:
- case Instruction::Opcode::FPTrunc:
- case Instruction::Opcode::BitCast: {
- assert(Operands.size() == 1u && "Casts are unary!");
- return CastInst::create(VecTy, Opcode, Operands[0], WhereIt, Ctx,
- "VCast");
- }
- case Instruction::Opcode::FCmp:
- case Instruction::Opcode::ICmp: {
- auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
- assert(all_of(drop_begin(Bndl),
- [Pred](auto *SBV) {
- return cast<CmpInst>(SBV)->getPredicate() == Pred;
- }) &&
- "Expected same predicate across bundle.");
- return CmpInst::create(Pred, Operands[0], Operands[1], WhereIt, Ctx,
- "VCmp");
- }
- case Instruction::Opcode::Select: {
- return SelectInst::create(Operands[0], Operands[1], Operands[2], WhereIt,
- Ctx, "Vec");
- }
- case Instruction::Opcode::FNeg: {
- auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
- auto OpC = UOp0->getOpcode();
- return UnaryOperator::createWithCopiedFlags(OpC, Operands[0], UOp0,
- WhereIt, Ctx, "Vec");
- }
- case Instruction::Opcode::Add:
- case Instruction::Opcode::FAdd:
- case Instruction::Opcode::Sub:
- case Instruction::Opcode::FSub:
- case Instruction::Opcode::Mul:
- case Instruction::Opcode::FMul:
- case Instruction::Opcode::UDiv:
- case Instruction::Opcode::SDiv:
- case Instruction::Opcode::FDiv:
- case Instruction::Opcode::URem:
- case Instruction::Opcode::SRem:
- case Instruction::Opcode::FRem:
- case Instruction::Opcode::Shl:
- case Instruction::Opcode::LShr:
- case Instruction::Opcode::AShr:
- case Instruction::Opcode::And:
- case Instruction::Opcode::Or:
- case Instruction::Opcode::Xor: {
- auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
- auto *LHS = Operands[0];
- auto *RHS = Operands[1];
- return BinaryOperator::createWithCopiedFlags(
- BinOp0->getOpcode(), LHS, RHS, BinOp0, WhereIt, Ctx, "Vec");
- }
- case Instruction::Opcode::Load: {
- auto *Ld0 = cast<LoadInst>(Bndl[0]);
- Value *Ptr = Ld0->getPointerOperand();
- return LoadInst::create(VecTy, Ptr, Ld0->getAlign(), WhereIt, Ctx,
- "VecL");
- }
- case Instruction::Opcode::Store: {
- auto Align = cast<StoreInst>(Bndl[0])->getAlign();
- Value *Val = Operands[0];
- Value *Ptr = Operands[1];
- return StoreInst::create(Val, Ptr, Align, WhereIt, Ctx);
- }
- case Instruction::Opcode::UncondBr:
- case Instruction::Opcode::CondBr:
- case Instruction::Opcode::Ret:
- case Instruction::Opcode::PHI:
- case Instruction::Opcode::AddrSpaceCast:
- case Instruction::Opcode::Call:
- case Instruction::Opcode::GetElementPtr:
- llvm_unreachable("Unimplemented");
- break;
- default:
- llvm_unreachable("Unimplemented");
- break;
- }
- llvm_unreachable("Missing switch case!");
- // TODO: Propagate debug info.
- };
+ Type *ScalarTy = VecUtils::getElementType(Utils::getExpectedType(Bndl[0]));
+ auto *VecTy = VecUtils::getWideType(ScalarTy, VecUtils::getNumLanes(Bndl));
- auto *NewI = CreateVectorInstr(Bndl, Operands);
- LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewI << "\n");
- return NewI;
-}
+ BasicBlock::iterator WhereIt =
+ getInsertPointAfterInstrs(Bndl, cast<Instruction>(Bndl[0])->getParent());
-void BottomUpVec::tryEraseDeadInstrs() {
- DenseMap<BasicBlock *, SmallVector<Instruction *>> SortedDeadInstrCandidates;
- // The dead instrs could span BBs, so we need to collect and sort them per BB.
- for (auto *DeadI : DeadInstrCandidates)
- SortedDeadInstrCandidates[DeadI->getParent()].push_back(DeadI);
- for (auto &Pair : SortedDeadInstrCandidates)
- sort(Pair.second,
- [](Instruction *I1, Instruction *I2) { return I1->comesBefore(I2); });
- for (const auto &Pair : SortedDeadInstrCandidates) {
- for (Instruction *I : reverse(Pair.second)) {
- if (I->hasNUses(0)) {
- // Erase the dead instructions bottom-to-top.
- LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Erase dead: " << *I << "\n");
- I->eraseFromParent();
- }
- }
+ auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
+ Value *NewI = nullptr;
+ switch (Opcode) {
+ case Instruction::Opcode::ZExt:
+ case Instruction::Opcode::SExt:
+ case Instruction::Opcode::FPToUI:
+ case Instruction::Opcode::FPToSI:
+ case Instruction::Opcode::FPExt:
+ case Instruction::Opcode::PtrToInt:
+ case Instruction::Opcode::IntToPtr:
+ case Instruction::Opcode::SIToFP:
+ case Instruction::Opcode::UIToFP:
+ case Instruction::Opcode::Trunc:
+ case Instruction::Opcode::FPTrunc:
+ case Instruction::Opcode::BitCast: {
+ assert(Operands.size() == 1u && "Casts are unary!");
+ NewI = CastInst::create(VecTy, Opcode, Operands[0], WhereIt, Ctx, "VCast");
+ break;
}
- DeadInstrCandidates.clear();
+ case Instruction::Opcode::FCmp:
+ case Instruction::Opcode::ICmp: {
+ auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
+ assert(all_of(drop_begin(Bndl),
+ [Pred](auto *SBV) {
+ return cast<CmpInst>(SBV)->getPredicate() == Pred;
+ }) &&
+ "Expected same predicate across bundle.");
+ NewI =
+ CmpInst::create(Pred, Operands[0], Operands[1], WhereIt, Ctx, "VCmp");
+ break;
+ }
+ case Instruction::Opcode::Select: {
+ NewI = SelectInst::create(Operands[0], Operands[1], Operands[2], WhereIt,
+ Ctx, "Vec");
+ break;
+ }
+ case Instruction::Opcode::FNeg: {
+ auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
+ auto OpC = UOp0->getOpcode();
+ NewI = UnaryOperator::createWithCopiedFlags(OpC, Operands[0], UOp0, WhereIt,
+ Ctx, "Vec");
+ break;
+ }
+ case Instruction::Opcode::Add:
+ case Instruction::Opcode::FAdd:
+ case Instruction::Opcode::Sub:
+ case Instruction::Opcode::FSub:
+ case Instruction::Opcode::Mul:
+ case Instruction::Opcode::FMul:
+ case Instruction::Opcode::UDiv:
+ case Instruction::Opcode::SDiv:
+ case Instruction::Opcode::FDiv:
+ case Instruction::Opcode::URem:
+ case Instruction::Opcode::SRem:
+ case Instruction::Opcode::FRem:
+ case Instruction::Opcode::Shl:
+ case Instruction::Opcode::LShr:
+ case Instruction::Opcode::AShr:
+ case Instruction::Opcode::And:
+ case Instruction::Opcode::Or:
+ case Instruction::Opcode::Xor: {
+ auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
+ auto *LHS = Operands[0];
+ auto *RHS = Operands[1];
+ NewI = BinaryOperator::createWithCopiedFlags(BinOp0->getOpcode(), LHS, RHS,
+ BinOp0, WhereIt, Ctx, "Vec");
+ break;
+ }
+ case Instruction::Opcode::Load: {
+ auto *Ld0 = cast<LoadInst>(Bndl[0]);
+ Value *Ptr = Ld0->getPointerOperand();
+ NewI = LoadInst::create(VecTy, Ptr, Ld0->getAlign(), WhereIt, Ctx, "VecL");
+ break;
+ }
+ case Instruction::Opcode::Store: {
+ auto Align = cast<StoreInst>(Bndl[0])->getAlign();
+ Value *Val = Operands[0];
+ Value *Ptr = Operands[1];
+ NewI = StoreInst::create(Val, Ptr, Align, WhereIt, Ctx);
+ break;
+ }
+ case Instruction::Opcode::UncondBr:
+ case Instruction::Opcode::CondBr:
+ case Instruction::Opcode::Ret:
+ case Instruction::Opcode::PHI:
+ case Instruction::Opcode::AddrSpaceCast:
+ case Instruction::Opcode::Call:
+ case Instruction::Opcode::GetElementPtr:
+ llvm_unreachable("Unimplemented");
+ break;
+ default:
+ llvm_unreachable("Unimplemented");
+ break;
+ }
+ return NewI;
}
-Value *BottomUpVec::createShuffle(Value *VecOp, const ShuffleMask &Mask,
- BasicBlock *UserBB) {
+Value *BundleVecPass::createShuffle(Value *VecOp, const ShuffleMask &Mask,
+ BasicBlock *UserBB) {
BasicBlock::iterator WhereIt = getInsertPointAfterInstrs({VecOp}, UserBB);
return ShuffleVectorInst::create(VecOp, VecOp, Mask, WhereIt,
VecOp->getContext(), "VShuf");
}
-Value *BottomUpVec::createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB) {
+Value *BundleVecPass::createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB) {
BasicBlock::iterator WhereIt = getInsertPointAfterInstrs(ToPack, UserBB);
Type *ScalarTy = VecUtils::getCommonScalarType(ToPack);
unsigned Lanes = VecUtils::getNumLanes(ToPack);
Type *VecTy = VecUtils::getWideType(ScalarTy, Lanes);
- // Create a series of pack instructions.
Value *LastInsert = PoisonValue::get(VecTy);
-
Context &Ctx = ToPack[0]->getContext();
-
unsigned InsertIdx = 0;
+
for (Value *Elm : ToPack) {
- // An element can be either scalar or vector. We need to generate different
- // IR for each case.
if (Elm->getType()->isVectorTy()) {
unsigned NumElms =
cast<FixedVectorType>(Elm->getType())->getNumElements();
for (auto ExtrLane : seq<int>(0, NumElms)) {
- // We generate extract-insert pairs, for each lane in `Elm`.
Constant *ExtrLaneC =
ConstantInt::getSigned(Type::getInt32Ty(Ctx), ExtrLane);
- // This may return a Constant if Elm is a Constant.
auto *ExtrI =
ExtractElementInst::create(Elm, ExtrLaneC, WhereIt, Ctx, "VPack");
if (!isa<Constant>(ExtrI))
WhereIt = std::next(cast<Instruction>(ExtrI)->getIterator());
Constant *InsertLaneC =
ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
- // This may also return a Constant if ExtrI is a Constant.
- auto *InsertI = InsertElementInst::create(
- LastInsert, ExtrI, InsertLaneC, WhereIt, Ctx, "VPack");
- LastInsert = InsertI;
- if (!isa<Constant>(InsertI))
+ LastInsert = InsertElementInst::create(LastInsert, ExtrI, InsertLaneC,
+ WhereIt, Ctx, "VPack");
+ if (!isa<Constant>(LastInsert))
WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
}
} else {
Constant *InsertLaneC =
ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
- // This may be folded into a Constant if LastInsert is a Constant. In
- // that case we only collect the last constant.
LastInsert = InsertElementInst::create(LastInsert, Elm, InsertLaneC,
- WhereIt, Ctx, "Pack");
- if (auto *NewI = dyn_cast<Instruction>(LastInsert))
- WhereIt = std::next(NewI->getIterator());
+ WhereIt, Ctx, "VPack");
+ if (!isa<Constant>(LastInsert))
+ WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
}
}
return LastInsert;
}
-void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
+void BundleVecPass::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
for (Value *V : Bndl)
DeadInstrCandidates.insert(cast<Instruction>(V));
- // Also collect the GEPs of vectorized loads and stores.
auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
switch (Opcode) {
case Instruction::Opcode::Load: {
@@ -278,16 +223,122 @@ void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
}
}
-Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
- ArrayRef<Value *> UserBndl, unsigned Depth,
- LegalityAnalysis &Legality) {
- bool StopForDebug =
- DebugBndlCnt++ >= StopBundle && StopBundle != StopBundleDisabled;
+void BundleVecPass::tryEraseDeadInstrs(StringRef PassName) {
+ DenseMap<BasicBlock *, SmallVector<Instruction *>> SortedDeadInstrCandidates;
+ for (auto *DeadI : DeadInstrCandidates)
+ SortedDeadInstrCandidates[DeadI->getParent()].push_back(DeadI);
+ for (auto &Pair : SortedDeadInstrCandidates)
+ sort(Pair.second,
+ [](Instruction *I1, Instruction *I2) { return I1->comesBefore(I2); });
+ for (const auto &Pair : SortedDeadInstrCandidates) {
+ for (Instruction *I : reverse(Pair.second)) {
+ if (I->hasNUses(0)) {
+ LLVM_DEBUG(dbgs() << PassName << "Erase dead: " << *I << "\n");
+ I->eraseFromParent();
+ }
+ }
+ }
+ DeadInstrCandidates.clear();
+}
+
+void BundleVecPass::emitUnpacksForExternalUses(ArrayRef<Value *> Bndl,
+ Value *Vec) {
+ BasicBlock::iterator WhereIt;
+ if (auto *VecI = dyn_cast<Instruction>(Vec)) {
+ WhereIt = std::next(VecI->getIterator());
+ } else {
+ // If Vec is a constant then it should be safe to emit the unpacks at the
+ // top of the block.
+ // Note: Extracts from constants are usually folded to constants.
+ assert(isa<Constant>(Vec) && "Expected constant!");
+ assert(isa<Instruction>(Bndl[0]) &&
+ "A widened Bndl should contain instrs!");
+ BasicBlock *BB = cast<Instruction>(Bndl[0])->getParent();
+ WhereIt =
+ BB->empty()
+ ? BB->begin()
+ : std::next(
+ VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
+ }
+
+ for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
+ for (User *U : Elm->users()) {
+ // Skip users that we just vectorized.
+ if (IMaps->isVectorized(U))
+ continue;
+ auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
+ Elm->replaceAllUsesWith(LastUnpackV);
+ }
+ }
+}
+
+Action *BundleVecPass::vectorizeRec(ArrayRef<Value *> Bndl,
+ ArrayRef<Value *> UserBndl, unsigned Depth,
+ LegalityAnalysis &Legality) {
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
VecUtils::dump(Bndl));
- const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
- : Legality.canVectorize(Bndl);
+
+ const LegalityResult &LegalityRes = [&]() -> const LegalityResult & {
+ if (Direction == VecDirection::BottomUp) {
+ bool StopForDebug = DebugBndlCnt++ >= BottomUpStopBundle &&
+ BottomUpStopBundle != StopBundleDisabled;
+ if (StopForDebug)
+ return Legality.getForcedPackForDebugging();
+ return Legality.canVectorize(Bndl);
+ }
+ return Legality.canVectorize(Bndl, /*SkipScheduling=*/true);
+ }();
+
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
+
+ if (Direction == VecDirection::TopDown) {
+ auto ActionPtr = std::make_unique<Action>(&LegalityRes, Bndl,
+ ArrayRef<Value *>(), Depth);
+
+ if (LegalityRes.getSubclassID() == LegalityResultID::Widen)
+ IMaps->registerVector(Bndl, ActionPtr.get());
+
+ // Pre-order push so defs are before uses.
+ Action *Action = ActionPtr.get();
+ Actions.push_back(std::move(ActionPtr));
+
+ if (LegalityRes.getSubclassID() == LegalityResultID::Widen) {
+ // Find users of Bndl to recurse down.
+ // Group the first user of each element if they match.
+ SmallVector<Value *, 4> NextUserBndl;
+ bool CanFormUserBndl = true;
+ for (Value *V : Bndl) {
+ if (V->user_begin() == V->user_end()) {
+ CanFormUserBndl = false;
+ break;
+ }
+ NextUserBndl.push_back(*V->user_begin());
+ }
+
+ if (CanFormUserBndl) {
+ auto *U0 = dyn_cast<Instruction>(NextUserBndl[0]);
+ if (!U0 || IMaps->isVectorized(U0))
+ CanFormUserBndl = false;
+ else {
+ for (Value *U : drop_begin(NextUserBndl)) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || UI->getOpcode() != U0->getOpcode() ||
+ UI->getType() != U0->getType() || IMaps->isVectorized(UI)) {
+ CanFormUserBndl = false;
+ break;
+ }
+ }
+ }
+ }
+
+ if (CanFormUserBndl)
+ vectorizeRec(NextUserBndl, {}, Depth + 1, Legality);
+ }
+
+ return Action;
+ }
+
+ // Bottom up direction
auto ActionPtr =
std::make_unique<Action>(&LegalityRes, Bndl, UserBndl, Depth);
SmallVector<Action *> Operands;
@@ -330,87 +381,64 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
return Action;
}
-#ifndef NDEBUG
-void BottomUpVec::ActionsVector::print(raw_ostream &OS) const {
- for (auto [Idx, Action] : enumerate(Actions)) {
- Action->print(OS);
- OS << "\n";
- }
-}
-void BottomUpVec::ActionsVector::dump() const { print(dbgs()); }
-#endif // NDEBUG
-
-void BottomUpVec::emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl,
- Value *Vec) {
- // Find where we should emit the unpacks.
- BasicBlock::iterator WhereIt;
- if (auto *VecI = dyn_cast<Instruction>(Vec)) {
- WhereIt = std::next(VecI->getIterator());
- } else {
- // If Vec is a constant then it should be safe to emit the unpacks at the
- // top of the block.
- // Note: Extracts from constants are usually folded to constants.
- assert(isa<Constant>(Vec) && "Expected constant!");
- assert(isa<Instruction>(Bndl[0]) &&
- "A widened Bndl should contain instrs!");
- BasicBlock *BB = cast<Instruction>(Bndl[0])->getParent();
- WhereIt =
- BB->empty()
- ? BB->begin()
- : std::next(
- VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
- }
-
- for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
- for (User *U : Elm->users()) {
- // Skip users that we just vectorized.
- if (IMaps->isVectorized(U))
- continue;
- auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
- Elm->replaceAllUsesWith(LastUnpackV);
- }
- }
-}
-
-Value *BottomUpVec::emitVectors() {
+Value *BundleVecPass::emitVectors() {
Value *NewVec = nullptr;
for (const auto &ActionPtr : Actions) {
ArrayRef<Value *> Bndl = ActionPtr->Bndl;
ArrayRef<Value *> UserBndl = ActionPtr->UserBndl;
const LegalityResult &LegalityRes = *ActionPtr->LegalityRes;
unsigned Depth = ActionPtr->Depth;
- auto *UserBB = !UserBndl.empty()
- ? cast<Instruction>(UserBndl.front())->getParent()
- : cast<Instruction>(Bndl[0])->getParent();
+ BasicBlock *UserBB = !UserBndl.empty()
+ ? cast<Instruction>(UserBndl[0])->getParent()
+ : cast<Instruction>(Bndl[0])->getParent();
switch (LegalityRes.getSubclassID()) {
case LegalityResultID::Widen: {
auto *I = cast<Instruction>(Bndl[0]);
SmallVector<Value *, 2> VecOperands;
- switch (I->getOpcode()) {
- case Instruction::Opcode::Load:
- VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
- break;
- case Instruction::Opcode::Store: {
- VecOperands.push_back(ActionPtr->Operands[0]->Vec);
- VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
- break;
- }
- default:
- // Visit all operands.
- for (Action *OpA : ActionPtr->Operands) {
- auto *VecOp = OpA->Vec;
- VecOperands.push_back(VecOp);
+ if (Direction == VecDirection::TopDown) {
+ switch (I->getOpcode()) {
+ case Instruction::Opcode::Load:
+ VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+ break;
+ case Instruction::Opcode::Store: {
+ auto OpBndl = getOperand(Bndl, 0);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+ VecOperands.push_back(OpA->Vec);
+ else
+ VecOperands.push_back(createPack(OpBndl, UserBB));
+ VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+ break;
+ }
+ default:
+ for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) {
+ SmallVector<Value *, 4> OpBndl = getOperand(Bndl, OpIdx);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+ VecOperands.push_back(OpA->Vec);
+ else
+ VecOperands.push_back(createPack(OpBndl, UserBB));
+ }
+ break;
+ }
+ } else {
+ switch (I->getOpcode()) {
+ case Instruction::Opcode::Load:
+ VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+ break;
+ case Instruction::Opcode::Store:
+ VecOperands.push_back(ActionPtr->Operands[0]->Vec);
+ VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+ break;
+ default:
+ for (Action *OpA : ActionPtr->Operands)
+ VecOperands.push_back(OpA->Vec);
+ break;
}
- break;
}
NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
- // Collect any potentially dead scalar instructions, including the
- // original scalars and pointer operands of loads/stores.
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewVec << "\n");
if (NewVec != nullptr)
collectPotentiallyDeadInstrs(Bndl);
-
- // Emit unpacks for all external uses, if any.
emitUnpacksForExternalUses(ActionPtr->Bndl, NewVec);
break;
}
@@ -423,8 +451,9 @@ Value *BottomUpVec::emitVectors() {
const ShuffleMask &Mask =
cast<DiamondReuseWithShuffle>(LegalityRes).getMask();
NewVec = createShuffle(VecOp, Mask, UserBB);
- assert(NewVec->getType() == VecOp->getType() &&
- "Expected same type! Bad mask ?");
+ if (Direction == VecDirection::BottomUp)
+ assert(NewVec->getType() == VecOp->getType() &&
+ "Expected same type! Bad mask ?");
break;
}
case LegalityResultID::DiamondReuseMultiInput: {
@@ -432,13 +461,12 @@ Value *BottomUpVec::emitVectors() {
cast<DiamondReuseMultiInput>(LegalityRes).getCollectDescr();
Type *ResTy = VecUtils::getWideType(Bndl[0]->getType(), Bndl.size());
- // TODO: Try to get WhereIt without creating a vector.
SmallVector<Value *, 4> DescrInstrs;
for (const auto &ElmDescr : Descr.getDescrs()) {
auto *V = ElmDescr.needsExtract() ? ElmDescr.getValue()->Vec
: ElmDescr.getScalar();
- if (auto *I = dyn_cast<Instruction>(V))
- DescrInstrs.push_back(I);
+ if (auto *Inst = dyn_cast<Instruction>(V))
+ DescrInstrs.push_back(Inst);
}
BasicBlock::iterator WhereIt =
getInsertPointAfterInstrs(DescrInstrs, UserBB);
@@ -447,10 +475,9 @@ Value *BottomUpVec::emitVectors() {
Context &Ctx = LastV->getContext();
unsigned Lane = 0;
for (const auto &ElmDescr : Descr.getDescrs()) {
- Value *VecOp = nullptr;
Value *ValueToInsert;
if (ElmDescr.needsExtract()) {
- VecOp = ElmDescr.getValue()->Vec;
+ Value *VecOp = ElmDescr.getValue()->Vec;
ConstantInt *IdxC =
ConstantInt::get(Type::getInt32Ty(Ctx), ElmDescr.getExtractIdx());
ValueToInsert = ExtractElementInst::create(
@@ -460,19 +487,10 @@ Value *BottomUpVec::emitVectors() {
}
auto NumLanesToInsert = VecUtils::getNumLanes(ValueToInsert);
if (NumLanesToInsert == 1) {
- // If we are inserting a scalar element then we need a single insert.
- // %VIns = insert %DstVec, %SrcScalar, Lane
ConstantInt *LaneC = ConstantInt::get(Type::getInt32Ty(Ctx), Lane);
LastV = InsertElementInst::create(LastV, ValueToInsert, LaneC,
WhereIt, Ctx, "VIns");
} else {
- // If we are inserting a vector element then we need to extract and
- // insert each vector element one by one with a chain of extracts and
- // inserts, for example:
- // %VExt0 = extract %SrcVec, 0
- // %VIns0 = insert %DstVec, %Vect0, Lane + 0
- // %VExt1 = extract %SrcVec, 1
- // %VIns1 = insert %VIns0, %Vect0, Lane + 1
for (unsigned LnCnt = 0; LnCnt != NumLanesToInsert; ++LnCnt) {
auto *ExtrIdxC = ConstantInt::get(Type::getInt32Ty(Ctx), LnCnt);
auto *ExtrI = ExtractElementInst::create(ValueToInsert, ExtrIdxC,
@@ -489,7 +507,6 @@ Value *BottomUpVec::emitVectors() {
break;
}
case LegalityResultID::Pack: {
- // If we can't vectorize the seeds then just return.
if (Depth == 0)
return nullptr;
NewVec = createPack(Bndl, UserBB);
@@ -502,11 +519,9 @@ Value *BottomUpVec::emitVectors() {
}
#ifndef NDEBUG
if (AlwaysVerify) {
- // This helps find broken IR by constantly verifying the function. Note
- // that this is very expensive and should only be used for debugging.
- Instruction *I0 = isa<Instruction>(Bndl[0])
- ? cast<Instruction>(Bndl[0])
- : cast<Instruction>(UserBndl[0]);
+ Instruction *I0 =
+ (isa<Instruction>(Bndl[0]) ? cast<Instruction>(Bndl[0])
+ : cast<Instruction>(UserBndl[0]));
assert(!Utils::verifyFunction(I0->getParent()->getParent(), dbgs()) &&
"Broken function!");
}
@@ -515,40 +530,42 @@ Value *BottomUpVec::emitVectors() {
return NewVec;
}
-bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl,
- LegalityAnalysis &Legality) {
+bool BundleVecPass::shouldSkipVectorization() {
+ return InvocationCnt++ >= StopAt && StopAt != StopAtDisabled;
+}
+
+bool BundleVecPass::tryVectorize(ArrayRef<Value *> Bndl,
+ LegalityAnalysis &Legality) {
Change = false;
- if (LLVM_UNLIKELY(BottomUpInvocationCnt++ >= StopAt &&
- StopAt != StopAtDisabled))
+ if (LLVM_UNLIKELY(shouldSkipVectorization()))
return false;
DeadInstrCandidates.clear();
Legality.clear();
Actions.clear();
DebugBndlCnt = 0;
vectorizeRec(Bndl, {}, /*Depth=*/0, Legality);
- LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "BottomUpVec: Vectorization Actions:\n";
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX
+ << (Direction == VecDirection::TopDown ? "TopDownVec"
+ : "BottomUpVec")
+ << ": Vectorization Actions:\n";
Actions.dump());
emitVectors();
- tryEraseDeadInstrs();
+ tryEraseDeadInstrs(DEBUG_PREFIX);
return Change;
}
-bool BottomUpVec::runOnRegion(Region &Rgn, const Analyses &A) {
+bool BundleVecPass::runOnRegion(Region &Rgn, const Analyses &A) {
const auto &SeedSlice = Rgn.getAux();
assert(SeedSlice.size() >= 2 && "Bad slice!");
+
Function &F = *SeedSlice[0]->getParent()->getParent();
IMaps = std::make_unique<InstrMaps>();
LegalityAnalysis Legality(A.getAA(), A.getScalarEvolution(),
F.getParent()->getDataLayout(), F.getContext(),
*IMaps);
- // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
- // Try to vectorize starting from the seed slice. The returned value
- // is true if we found vectorizable code and generated some vector
- // code for it. It does not mean that the code is profitable.
return tryVectorize(SeedSliceVals, Legality);
}
-} // namespace sandboxir
-} // namespace llvm
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index 10ba595910ee9..40d03c3886c0b 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -26,6 +26,7 @@ REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
REGION_PASS("tr-revert", ::llvm::sandboxir::TransactionAlwaysRevert)
REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
+REGION_PASS("top-down-vec", ::llvm::sandboxir::TopDownVec)
REGION_PASS("load-store-vec", ::llvm::sandboxir::LoadStoreVec)
#undef REGION_PASS
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 9326869461082..e53895660859c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -9,6 +9,7 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysRevert.h"
diff --git a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
index 593965ab01680..624cf3d918362 100644
--- a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
; Checks the handling of users outside the vectorized graph.
@@ -13,6 +14,20 @@ define void @external_users(ptr %ptr) {
; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
; CHECK-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub float [[LD1]], 0.000000e+00
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x float> [[VPACK]], float [[SUB1]], i32 1, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: store <2 x float> [[VPACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -35,6 +50,17 @@ define void @external_user_of_constant(ptr %ptr, ptr %ptrX) {
; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
; CHECK-NEXT: store i32 0, ptr [[PTRX]], align 4
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_user_of_constant(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], ptr [[PTRX:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[ZEXT0:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: [[ZEXT1:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i32> poison, i32 [[ZEXT0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i32> [[VPACK]], i32 [[ZEXT1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: store <2 x i32> [[VPACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: store i32 [[ZEXT0]], ptr [[PTRX]], align 4
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -57,6 +83,23 @@ define void @vector_external_users(ptr %ptr) {
; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACKINS2]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <3 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK2:%.*]] = insertelement <3 x float> [[VPACK]], float [[VPACK1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK3:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK4:%.*]] = insertelement <3 x float> [[VPACK2]], float [[VPACK3]], i32 2, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: store <3 x float> [[VPACK4]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -80,6 +123,26 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK-NEXT: store <4 x float> [[VEC]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACK]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users_lane_and_index_differ(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load <2 x float>, ptr [[PTR0]], align 8
+; TOPDOWN-NEXT: [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub <2 x float> [[LD0]], zeroinitializer
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT: [[VPACK:%.*]] = extractelement <2 x float> [[SUB0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <4 x float> poison, float [[VPACK]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK2:%.*]] = extractelement <2 x float> [[SUB0]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK3:%.*]] = insertelement <4 x float> [[VPACK1]], float [[VPACK2]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK4:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK5:%.*]] = insertelement <4 x float> [[VPACK3]], float [[VPACK4]], i32 2, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK6:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK7:%.*]] = insertelement <4 x float> [[VPACK5]], float [[VPACK6]], i32 3, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: store <4 x float> [[VPACK7]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr <2 x float>, ptr %ptr, i32 0
%ptr1 = getelementptr <2 x float>, ptr %ptr, i32 1
@@ -99,3 +162,8 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index 743d705fd48ff..2a50ba84e53db 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
@@ -7,6 +8,12 @@ define void @pack_constants(ptr %ptr) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @pack_constants(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
%ptr1 = getelementptr i8, ptr %ptr, i32 1
@@ -27,14 +34,31 @@ define void @packPHIs(ptr %ptr) {
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packPHIs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
br label %loop
@@ -63,14 +87,31 @@ define void @packFromOtherBB(ptr %ptr, i8 %val) {
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromOtherBB(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[VAL:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[VAL]], 0
+; TOPDOWN-NEXT: [[MUL1:%.*]] = mul i8 [[VAL]], 1
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %val, 0
%mul1 = mul i8 %val, 1
@@ -97,12 +138,25 @@ define void @packFromDiffBBs(ptr %ptr, i8 %v) {
; CHECK-NEXT: br label %[[BB:.*]]
; CHECK: [[BB]]:
; CHECK-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromDiffBBs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[V:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*:]]
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[V]], 1
+; TOPDOWN-NEXT: br label %[[BB:.*]]
+; TOPDOWN: [[BB]]:
+; TOPDOWN-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %v, 1
br label %bb
@@ -121,3 +175,8 @@ bb:
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
new file mode 100644
index 0000000000000..a9db7d9225685
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 \
+; RUN: -sbvec-collect-seeds=loads \
+; RUN: -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" \
+; RUN: %s -S | FileCheck %s
+
+define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_fadd_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META0]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META0]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define void @load_chain_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_chain_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fmul <2 x float> [[VECL]], splat (float 3.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[VEC1:%.*]] = fadd <2 x float> [[VEC]], splat (float 2.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC1]], ptr [[PTR2_0]], align 4, !sandboxvec [[META1]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fmul0 = fmul float %ld0, 3.0
+ %fmul1 = fmul float %ld1, 3.0
+
+ %fadd0 = fadd float %fmul0, 2.0
+ %fadd1 = fadd float %fmul1, 2.0
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define float @load_fadd_external_use(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define float @load_fadd_external_use(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VEC]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: store float [[UNPACK]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[UNPACK1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret float [[UNPACK]]
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+
+ ret float %fadd0
+}
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+; CHECK: [[META2]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index e8f04be8eb71d..613aac0501282 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -18,7 +18,7 @@ static_library("Vectorize") {
"SandboxVectorizer/InstrMaps.cpp",
"SandboxVectorizer/Interval.cpp",
"SandboxVectorizer/Legality.cpp",
- "SandboxVectorizer/Passes/BottomUpVec.cpp",
+ "SandboxVectorizer/Passes/BundleVecPass.cpp",
"SandboxVectorizer/Passes/LoadStoreVec.cpp",
"SandboxVectorizer/Passes/PackReuse.cpp",
"SandboxVectorizer/Passes/RegionsFromBBs.cpp",
More information about the llvm-commits
mailing list