[llvm] [SandboxVectorizer] Implement topdown vectorizer (PR #205249)

Anshil Gandhi via llvm-commits llvm-commits at lists.llvm.org
Tue Jun 23 22:55:59 PDT 2026


https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/205249

>From f06744bbb411f66b11e9f545a30b25d32265b392 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Sun, 14 Jun 2026 17:01:48 -0400
Subject: [PATCH] [SBVec] Implement topDown/botUp vectorizers in unison

This patch introduces the `top-down-vec` pass to the Sandbox Vectorizer,
adding the ability to traverse use-def chains top-down to discover and
collect vectorization opportunities. Furthermore, this patch unifies
the two vectorizers into a single implementation to minimize code
duplication.
---
 .../SandboxVectorizer/Passes/BottomUpVec.h    |  77 +--
 .../SandboxVectorizer/Passes/BundleVecPass.h  | 143 +++++
 .../SandboxVectorizer/Passes/TopDownVec.h     |  26 +
 .../Vectorize/SandboxVectorizer/VecUtils.h    |   8 +-
 llvm/lib/Transforms/Vectorize/CMakeLists.txt  |   2 +-
 .../{BottomUpVec.cpp => BundleVecPass.cpp}    | 599 +++++++++---------
 .../SandboxVectorizer/Passes/PassRegistry.def |   1 +
 .../SandboxVectorizerPassBuilder.cpp          |   1 +
 .../SandboxVectorizer/external_uses.ll        |  68 ++
 .../test/Transforms/SandboxVectorizer/pack.ll |  77 ++-
 .../SandboxVectorizer/topdown_vec.ll          |  94 +++
 .../llvm/lib/Transforms/Vectorize/BUILD.gn    |   2 +-
 12 files changed, 721 insertions(+), 377 deletions(-)
 create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h
 create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
 rename llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/{BottomUpVec.cpp => BundleVecPass.cpp} (50%)
 create mode 100644 llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll

diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index d024bb74ac0c9..08590add639fb 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -12,13 +12,7 @@
 #ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
 
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/SandboxIR/Constant.h"
-#include "llvm/SandboxIR/Pass.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h"
 
 namespace llvm::sandboxir {
 
@@ -32,74 +26,9 @@ namespace llvm::sandboxir {
 /// profitable or not. For now profitability is checked at the end of the region
 /// pass pipeline by a dedicated pass that accepts or rejects the IR
 /// transaction, depending on the cost.
-class LLVM_ABI BottomUpVec final : public RegionPass {
-  bool Change = false;
-  /// The original instructions that are potentially dead after vectorization.
-  DenseSet<Instruction *> DeadInstrCandidates;
-  /// Maps scalars to vectors.
-  std::unique_ptr<InstrMaps> IMaps;
-  /// Counter used for force-stopping the vectorizer after this many
-  /// invocations. Used for debugging miscompiles.
-  unsigned long BottomUpInvocationCnt = 0;
-
-  /// Creates and returns a vector instruction that replaces the instructions in
-  /// \p Bndl. \p Operands are the already vectorized operands.
-  Value *createVectorInstr(ArrayRef<Value *> Bndl, ArrayRef<Value *> Operands);
-  /// Erases all dead instructions from the dead instruction candidates
-  /// collected during vectorization.
-  void tryEraseDeadInstrs();
-  /// Creates a shuffle instruction that shuffles \p VecOp according to \p Mask.
-  /// \p UserBB is the block of the user bundle.
-  Value *createShuffle(Value *VecOp, const ShuffleMask &Mask,
-                       BasicBlock *UserBB);
-  /// Packs all elements of \p ToPack into a vector and returns that vector. \p
-  /// UserBB is the block of the user bundle.
-  Value *createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB);
-  /// After we create vectors for groups of instructions, the original
-  /// instructions are potentially dead and may need to be removed. This
-  /// function helps collect these instructions (along with the pointer operands
-  /// for loads/stores) so that they can be cleaned up later.
-  void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
-
-  /// Helper class describing how(if) to vectorize the code.
-  class ActionsVector {
-  private:
-    SmallVector<std::unique_ptr<Action>, 16> Actions;
-
-  public:
-    auto begin() const { return Actions.begin(); }
-    auto end() const { return Actions.end(); }
-    void push_back(std::unique_ptr<Action> &&ActPtr) {
-      ActPtr->Idx = Actions.size();
-      Actions.push_back(std::move(ActPtr));
-    }
-    void clear() { Actions.clear(); }
-#ifndef NDEBUG
-    void print(raw_ostream &OS) const;
-    void dump() const;
-#endif // NDEBUG
-  };
-  ActionsVector Actions;
-  /// Helper counter for debugging. It counts the bundles that we attempt to
-  /// vectorize in vectorizeRec().
-  unsigned DebugBndlCnt = 0;
-
-  /// Recursively try to vectorize \p Bndl and its operands. This populates the
-  /// `Actions` vector.
-  Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
-                       unsigned Depth, LegalityAnalysis &Legality);
-  /// If the values in \p Bndl have external users, then emit unpacks and
-  /// connect them to the users. \p Vec is the vectorized form of \p Bndl.
-  void emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl, Value *Vec);
-  /// Generate vector instructions based on `Actions` and return the last vector
-  /// created.
-  Value *emitVectors();
-  /// Entry point for vectorization starting from \p Seeds.
-  bool tryVectorize(ArrayRef<Value *> Seeds, LegalityAnalysis &Legality);
-
+class LLVM_ABI BottomUpVec final : public BundleVecPass {
 public:
-  BottomUpVec() : RegionPass("bottom-up-vec") {}
-  bool runOnRegion(Region &Rgn, const Analyses &A) final;
+  BottomUpVec() : BundleVecPass("bottom-up-vec", VecDirection::BottomUp) {}
 };
 
 } // namespace llvm::sandboxir
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h
new file mode 100644
index 0000000000000..63c9657014960
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h
@@ -0,0 +1,143 @@
+//===- BundleVecPass.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Region pass that vectorizes instruction bundles, either top-down or
+// bottom-up.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BUNDLEVECPASS_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BUNDLEVECPASS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+namespace llvm::sandboxir {
+
+enum class VecDirection { TopDown, BottomUp };
+
+/// Vectorizes instruction bundles starting from a seed slice in the region.
+/// The traversal direction is selected via \p VecDirection.
+class LLVM_ABI BundleVecPass : public RegionPass {
+protected:
+  VecDirection Direction;
+  /// Set to true whenever the pass emits vector code in the current region.
+  bool Change = false;
+  /// Scalar instructions that may be dead after vectorization.
+  DenseSet<Instruction *> DeadInstrCandidates;
+  /// Maps scalar values/instructions to their vector counterparts.
+  std::unique_ptr<InstrMaps> IMaps;
+  /// Helper counter for debugging — counts bundles attempted in vectorizeRec.
+  unsigned DebugBndlCnt = 0;
+
+  /// Ordered list of vectorization decisions produced by vectorizeRec.
+  class ActionsVector {
+    SmallVector<std::unique_ptr<Action>, 16> Actions;
+
+  public:
+    auto begin() const { return Actions.begin(); }
+    auto end() const { return Actions.end(); }
+    auto rbegin() const { return Actions.rbegin(); }
+    auto rend() const { return Actions.rend(); }
+    void push_back(std::unique_ptr<Action> &&ActPtr) {
+      ActPtr->Idx = Actions.size();
+      Actions.push_back(std::move(ActPtr));
+    }
+    void clear() { Actions.clear(); }
+#ifndef NDEBUG
+    void print(raw_ostream &OS) const {
+      for (auto [Idx, Action] : enumerate(Actions)) {
+        Action->print(OS);
+        OS << "\n";
+      }
+    }
+    LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
+#endif // NDEBUG
+  };
+  ActionsVector Actions;
+
+  explicit BundleVecPass(StringRef Name, VecDirection Dir)
+      : RegionPass(Name), Direction(Dir) {}
+
+  /// Recursively try to vectorize \p Bndl. For bottom-up vectorization \p
+  /// UserBndl tracks the bundle of users that led to this recursion.
+  Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
+                       unsigned Depth, LegalityAnalysis &Legality);
+
+  /// \Returns the operand at \p OpIdx for each instruction in \p Bndl.
+  static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
+                                            unsigned OpIdx) {
+    SmallVector<Value *, 4> Operands;
+    for (Value *BndlV : Bndl)
+      Operands.push_back(cast<Instruction>(BndlV)->getOperand(OpIdx));
+    return Operands;
+  }
+
+  /// \Returns the BB iterator after the lowest instruction in \p Vals, or the
+  /// top of BB (after any PHIs) if no instruction is found in \p Vals.
+  static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef<Value *> Vals,
+                                                        BasicBlock *BB) {
+    auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB));
+    if (BotI == nullptr)
+      return BB->empty() ? BB->begin()
+                         : std::next(VecUtils::getLastPHIOrSelf(&*BB->begin())
+                                         ->getIterator());
+    return std::next(BotI->getIterator());
+  }
+
+  /// Creates and returns a new vector instruction widening \p Bndl with
+  /// \p Operands as the vector operands.
+  static Value *createVectorInstr(ArrayRef<Value *> Bndl,
+                                  ArrayRef<Value *> Operands);
+
+  /// Creates a shuffle of \p VecOp according to \p Mask, inserted in \p UserBB.
+  static Value *createShuffle(Value *VecOp, const ShuffleMask &Mask,
+                              BasicBlock *UserBB);
+
+  /// Packs all scalars/vectors in \p ToPack into a single vector, inserted in
+  /// \p UserBB.
+  static Value *createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB);
+
+  /// Adds the instructions in \p Bndl (and pointer operands of loads/stores)
+  /// to DeadInstrCandidates for later cleanup.
+  void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
+
+  /// Erases all zero-use instructions from DeadInstrCandidates.
+  /// \p PassName is used as the debug-output prefix.
+  void tryEraseDeadInstrs(StringRef PassName);
+
+  /// For each element of \p Bndl that has external uses (users not yet
+  /// vectorized), emits extract instructions from \p Vec and replaces the
+  /// scalar uses with them.
+  void emitUnpacksForExternalUses(ArrayRef<Value *> Bndl, Value *Vec);
+
+  /// Generate vector instructions based on `Actions` and return the last vector
+  /// created.
+  Value *emitVectors();
+
+  /// \Returns true if vectorization should be skipped due to the stop-at limit.
+  bool shouldSkipVectorization();
+
+private:
+  unsigned long InvocationCnt = 0;
+
+  bool tryVectorize(ArrayRef<Value *> Bndl, LegalityAnalysis &Legality);
+
+public:
+  bool runOnRegion(Region &Rgn, const Analyses &A) override;
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BUNDLEVECPASS_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
new file mode 100644
index 0000000000000..2aebf6aae9e48
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
@@ -0,0 +1,26 @@
+//===- TopDownVec.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A Top-Down Vectorizer pass.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h"
+
+namespace llvm::sandboxir {
+
+class LLVM_ABI TopDownVec final : public BundleVecPass {
+public:
+  TopDownVec() : BundleVecPass("top-down-vec", VecDirection::TopDown) {}
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 99e411a1723b0..57fb884c27699 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -12,6 +12,8 @@
 #ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
 
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/SandboxIR/Type.h"
@@ -33,6 +35,10 @@ template <> struct DenseMapInfo<SmallVector<sandboxir::Value *>> {
 
 namespace sandboxir {
 
+class ShuffleMask;
+class InstrMaps;
+class BundleVecPass;
+
 class VecUtils {
 public:
   /// \Returns the number of elements in \p Ty. That is the number of lanes if a
@@ -379,7 +385,7 @@ class VecUtils {
   }
 
 #ifndef NDEBUG
-  /// Helper dump function for debugging.
+  /// Print \p Bndl. Dump function for debugging.
   LLVM_DUMP_METHOD static void dump(ArrayRef<Value *> Bndl);
   LLVM_DUMP_METHOD static void dump(ArrayRef<Instruction *> Bndl);
 #endif // NDEBUG
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index d081747b821a4..f823fed93c80f 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -8,7 +8,7 @@ add_llvm_component_library(LLVMVectorize
   SandboxVectorizer/InstrMaps.cpp
   SandboxVectorizer/Interval.cpp
   SandboxVectorizer/Legality.cpp
-  SandboxVectorizer/Passes/BottomUpVec.cpp
+  SandboxVectorizer/Passes/BundleVecPass.cpp
   SandboxVectorizer/Passes/LoadStoreVec.cpp
   SandboxVectorizer/Passes/PackReuse.cpp
   SandboxVectorizer/Passes/RegionsFromBBs.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.cpp
similarity index 50%
rename from llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
rename to llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.cpp
index 6bf257fcf8b1d..4a889fe1433fd 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.cpp
@@ -1,4 +1,4 @@
-//===- BottomUpVec.cpp - A bottom-up vectorizer pass ----------------------===//
+//===- BundleVecPass.cpp - Bundle vectorizer pass -------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,27 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVecPass.h"
 #include "llvm/SandboxIR/Function.h"
 #include "llvm/SandboxIR/Instruction.h"
 #include "llvm/SandboxIR/Module.h"
 #include "llvm/SandboxIR/Region.h"
 #include "llvm/SandboxIR/Utils.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Debug.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
 
 namespace llvm {
 
-#ifndef NDEBUG
-static cl::opt<bool>
-    AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
-                 cl::desc("Helps find bugs by verifying the IR whenever we "
-                          "emit new instructions (*very* expensive)."));
-#endif // NDEBUG
-
 static constexpr unsigned long StopAtDisabled =
     std::numeric_limits<unsigned long>::max();
+
 static cl::opt<unsigned long>
     StopAt("sbvec-stop-at", cl::init(StopAtDisabled), cl::Hidden,
            cl::desc("Vectorize if the invocation count is < than this. 0 "
@@ -34,229 +26,182 @@ static cl::opt<unsigned long>
 
 static constexpr unsigned long StopBundleDisabled =
     std::numeric_limits<unsigned long>::max();
+
 static cl::opt<unsigned long>
-    StopBundle("sbvec-stop-bndl", cl::init(StopBundleDisabled), cl::Hidden,
-               cl::desc("Vectorize up to this many bundles."));
+    BottomUpStopBundle("sbvec-stop-bndl", cl::init(StopBundleDisabled),
+                       cl::Hidden,
+                       cl::desc("Vectorize up to this many bundles."));
 
-namespace sandboxir {
+#ifndef NDEBUG
+static cl::opt<bool>
+    AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
+                 cl::desc("Helps find bugs by verifying the IR whenever "
+                          "we emit new instructions (*very* expensive)."));
+#endif // NDEBUG
+} // namespace llvm
 
-static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
-                                          unsigned OpIdx) {
-  SmallVector<Value *, 4> Operands;
-  for (Value *BndlV : Bndl) {
-    auto *BndlI = cast<Instruction>(BndlV);
-    Operands.push_back(BndlI->getOperand(OpIdx));
-  }
-  return Operands;
-}
+namespace llvm::sandboxir {
 
-/// \Returns the BB iterator after the lowest instruction in \p Vals, or the top
-/// of BB if no instruction found in \p Vals.
-static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef<Value *> Vals,
-                                                      BasicBlock *BB) {
-  auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB));
-  if (BotI == nullptr)
-    // We are using BB->begin() (or after PHIs) as the fallback insert point.
-    return BB->empty()
-               ? BB->begin()
-               : std::next(
-                     VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
-  return std::next(BotI->getIterator());
-}
+Value *BundleVecPass::createVectorInstr(ArrayRef<Value *> Bndl,
+                                        ArrayRef<Value *> Operands) {
+  assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
+         "Expect Instructions!");
+  auto &Ctx = Bndl[0]->getContext();
 
-Value *BottomUpVec::createVectorInstr(ArrayRef<Value *> Bndl,
-                                      ArrayRef<Value *> Operands) {
-  auto CreateVectorInstr = [](ArrayRef<Value *> Bndl,
-                              ArrayRef<Value *> Operands) -> Value * {
-    assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
-           "Expect Instructions!");
-    auto &Ctx = Bndl[0]->getContext();
-
-    Type *ScalarTy = VecUtils::getElementType(Utils::getExpectedType(Bndl[0]));
-    auto *VecTy = VecUtils::getWideType(ScalarTy, VecUtils::getNumLanes(Bndl));
-
-    BasicBlock::iterator WhereIt = getInsertPointAfterInstrs(
-        Bndl, cast<Instruction>(Bndl[0])->getParent());
-
-    auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
-    switch (Opcode) {
-    case Instruction::Opcode::ZExt:
-    case Instruction::Opcode::SExt:
-    case Instruction::Opcode::FPToUI:
-    case Instruction::Opcode::FPToSI:
-    case Instruction::Opcode::FPExt:
-    case Instruction::Opcode::PtrToInt:
-    case Instruction::Opcode::IntToPtr:
-    case Instruction::Opcode::SIToFP:
-    case Instruction::Opcode::UIToFP:
-    case Instruction::Opcode::Trunc:
-    case Instruction::Opcode::FPTrunc:
-    case Instruction::Opcode::BitCast: {
-      assert(Operands.size() == 1u && "Casts are unary!");
-      return CastInst::create(VecTy, Opcode, Operands[0], WhereIt, Ctx,
-                              "VCast");
-    }
-    case Instruction::Opcode::FCmp:
-    case Instruction::Opcode::ICmp: {
-      auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
-      assert(all_of(drop_begin(Bndl),
-                    [Pred](auto *SBV) {
-                      return cast<CmpInst>(SBV)->getPredicate() == Pred;
-                    }) &&
-             "Expected same predicate across bundle.");
-      return CmpInst::create(Pred, Operands[0], Operands[1], WhereIt, Ctx,
-                             "VCmp");
-    }
-    case Instruction::Opcode::Select: {
-      return SelectInst::create(Operands[0], Operands[1], Operands[2], WhereIt,
-                                Ctx, "Vec");
-    }
-    case Instruction::Opcode::FNeg: {
-      auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
-      auto OpC = UOp0->getOpcode();
-      return UnaryOperator::createWithCopiedFlags(OpC, Operands[0], UOp0,
-                                                  WhereIt, Ctx, "Vec");
-    }
-    case Instruction::Opcode::Add:
-    case Instruction::Opcode::FAdd:
-    case Instruction::Opcode::Sub:
-    case Instruction::Opcode::FSub:
-    case Instruction::Opcode::Mul:
-    case Instruction::Opcode::FMul:
-    case Instruction::Opcode::UDiv:
-    case Instruction::Opcode::SDiv:
-    case Instruction::Opcode::FDiv:
-    case Instruction::Opcode::URem:
-    case Instruction::Opcode::SRem:
-    case Instruction::Opcode::FRem:
-    case Instruction::Opcode::Shl:
-    case Instruction::Opcode::LShr:
-    case Instruction::Opcode::AShr:
-    case Instruction::Opcode::And:
-    case Instruction::Opcode::Or:
-    case Instruction::Opcode::Xor: {
-      auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
-      auto *LHS = Operands[0];
-      auto *RHS = Operands[1];
-      return BinaryOperator::createWithCopiedFlags(
-          BinOp0->getOpcode(), LHS, RHS, BinOp0, WhereIt, Ctx, "Vec");
-    }
-    case Instruction::Opcode::Load: {
-      auto *Ld0 = cast<LoadInst>(Bndl[0]);
-      Value *Ptr = Ld0->getPointerOperand();
-      return LoadInst::create(VecTy, Ptr, Ld0->getAlign(), WhereIt, Ctx,
-                              "VecL");
-    }
-    case Instruction::Opcode::Store: {
-      auto Align = cast<StoreInst>(Bndl[0])->getAlign();
-      Value *Val = Operands[0];
-      Value *Ptr = Operands[1];
-      return StoreInst::create(Val, Ptr, Align, WhereIt, Ctx);
-    }
-    case Instruction::Opcode::UncondBr:
-    case Instruction::Opcode::CondBr:
-    case Instruction::Opcode::Ret:
-    case Instruction::Opcode::PHI:
-    case Instruction::Opcode::AddrSpaceCast:
-    case Instruction::Opcode::Call:
-    case Instruction::Opcode::GetElementPtr:
-      llvm_unreachable("Unimplemented");
-      break;
-    default:
-      llvm_unreachable("Unimplemented");
-      break;
-    }
-    llvm_unreachable("Missing switch case!");
-    // TODO: Propagate debug info.
-  };
+  Type *ScalarTy = VecUtils::getElementType(Utils::getExpectedType(Bndl[0]));
+  auto *VecTy = VecUtils::getWideType(ScalarTy, VecUtils::getNumLanes(Bndl));
 
-  auto *NewI = CreateVectorInstr(Bndl, Operands);
-  LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewI << "\n");
-  return NewI;
-}
+  BasicBlock::iterator WhereIt =
+      getInsertPointAfterInstrs(Bndl, cast<Instruction>(Bndl[0])->getParent());
 
-void BottomUpVec::tryEraseDeadInstrs() {
-  DenseMap<BasicBlock *, SmallVector<Instruction *>> SortedDeadInstrCandidates;
-  // The dead instrs could span BBs, so we need to collect and sort them per BB.
-  for (auto *DeadI : DeadInstrCandidates)
-    SortedDeadInstrCandidates[DeadI->getParent()].push_back(DeadI);
-  for (auto &Pair : SortedDeadInstrCandidates)
-    sort(Pair.second,
-         [](Instruction *I1, Instruction *I2) { return I1->comesBefore(I2); });
-  for (const auto &Pair : SortedDeadInstrCandidates) {
-    for (Instruction *I : reverse(Pair.second)) {
-      if (I->hasNUses(0)) {
-        // Erase the dead instructions bottom-to-top.
-        LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Erase dead: " << *I << "\n");
-        I->eraseFromParent();
-      }
-    }
+  auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
+  Value *NewI = nullptr;
+  switch (Opcode) {
+  case Instruction::Opcode::ZExt:
+  case Instruction::Opcode::SExt:
+  case Instruction::Opcode::FPToUI:
+  case Instruction::Opcode::FPToSI:
+  case Instruction::Opcode::FPExt:
+  case Instruction::Opcode::PtrToInt:
+  case Instruction::Opcode::IntToPtr:
+  case Instruction::Opcode::SIToFP:
+  case Instruction::Opcode::UIToFP:
+  case Instruction::Opcode::Trunc:
+  case Instruction::Opcode::FPTrunc:
+  case Instruction::Opcode::BitCast: {
+    assert(Operands.size() == 1u && "Casts are unary!");
+    NewI = CastInst::create(VecTy, Opcode, Operands[0], WhereIt, Ctx, "VCast");
+    break;
   }
-  DeadInstrCandidates.clear();
+  case Instruction::Opcode::FCmp:
+  case Instruction::Opcode::ICmp: {
+    auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
+    assert(all_of(drop_begin(Bndl),
+                  [Pred](auto *SBV) {
+                    return cast<CmpInst>(SBV)->getPredicate() == Pred;
+                  }) &&
+           "Expected same predicate across bundle.");
+    NewI =
+        CmpInst::create(Pred, Operands[0], Operands[1], WhereIt, Ctx, "VCmp");
+    break;
+  }
+  case Instruction::Opcode::Select: {
+    NewI = SelectInst::create(Operands[0], Operands[1], Operands[2], WhereIt,
+                              Ctx, "Vec");
+    break;
+  }
+  case Instruction::Opcode::FNeg: {
+    auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
+    auto OpC = UOp0->getOpcode();
+    NewI = UnaryOperator::createWithCopiedFlags(OpC, Operands[0], UOp0, WhereIt,
+                                                Ctx, "Vec");
+    break;
+  }
+  case Instruction::Opcode::Add:
+  case Instruction::Opcode::FAdd:
+  case Instruction::Opcode::Sub:
+  case Instruction::Opcode::FSub:
+  case Instruction::Opcode::Mul:
+  case Instruction::Opcode::FMul:
+  case Instruction::Opcode::UDiv:
+  case Instruction::Opcode::SDiv:
+  case Instruction::Opcode::FDiv:
+  case Instruction::Opcode::URem:
+  case Instruction::Opcode::SRem:
+  case Instruction::Opcode::FRem:
+  case Instruction::Opcode::Shl:
+  case Instruction::Opcode::LShr:
+  case Instruction::Opcode::AShr:
+  case Instruction::Opcode::And:
+  case Instruction::Opcode::Or:
+  case Instruction::Opcode::Xor: {
+    auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
+    auto *LHS = Operands[0];
+    auto *RHS = Operands[1];
+    NewI = BinaryOperator::createWithCopiedFlags(BinOp0->getOpcode(), LHS, RHS,
+                                                 BinOp0, WhereIt, Ctx, "Vec");
+    break;
+  }
+  case Instruction::Opcode::Load: {
+    auto *Ld0 = cast<LoadInst>(Bndl[0]);
+    Value *Ptr = Ld0->getPointerOperand();
+    NewI = LoadInst::create(VecTy, Ptr, Ld0->getAlign(), WhereIt, Ctx, "VecL");
+    break;
+  }
+  case Instruction::Opcode::Store: {
+    auto Align = cast<StoreInst>(Bndl[0])->getAlign();
+    Value *Val = Operands[0];
+    Value *Ptr = Operands[1];
+    NewI = StoreInst::create(Val, Ptr, Align, WhereIt, Ctx);
+    break;
+  }
+  case Instruction::Opcode::UncondBr:
+  case Instruction::Opcode::CondBr:
+  case Instruction::Opcode::Ret:
+  case Instruction::Opcode::PHI:
+  case Instruction::Opcode::AddrSpaceCast:
+  case Instruction::Opcode::Call:
+  case Instruction::Opcode::GetElementPtr:
+    llvm_unreachable("Unimplemented");
+    break;
+  default:
+    llvm_unreachable("Unimplemented");
+    break;
+  }
+  return NewI;
 }
 
-Value *BottomUpVec::createShuffle(Value *VecOp, const ShuffleMask &Mask,
-                                  BasicBlock *UserBB) {
+Value *BundleVecPass::createShuffle(Value *VecOp, const ShuffleMask &Mask,
+                                    BasicBlock *UserBB) {
   BasicBlock::iterator WhereIt = getInsertPointAfterInstrs({VecOp}, UserBB);
   return ShuffleVectorInst::create(VecOp, VecOp, Mask, WhereIt,
                                    VecOp->getContext(), "VShuf");
 }
 
-Value *BottomUpVec::createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB) {
+Value *BundleVecPass::createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB) {
   BasicBlock::iterator WhereIt = getInsertPointAfterInstrs(ToPack, UserBB);
 
   Type *ScalarTy = VecUtils::getCommonScalarType(ToPack);
   unsigned Lanes = VecUtils::getNumLanes(ToPack);
   Type *VecTy = VecUtils::getWideType(ScalarTy, Lanes);
 
-  // Create a series of pack instructions.
   Value *LastInsert = PoisonValue::get(VecTy);
-
   Context &Ctx = ToPack[0]->getContext();
-
   unsigned InsertIdx = 0;
+
   for (Value *Elm : ToPack) {
-    // An element can be either scalar or vector. We need to generate different
-    // IR for each case.
     if (Elm->getType()->isVectorTy()) {
       unsigned NumElms =
           cast<FixedVectorType>(Elm->getType())->getNumElements();
       for (auto ExtrLane : seq<int>(0, NumElms)) {
-        // We generate extract-insert pairs, for each lane in `Elm`.
         Constant *ExtrLaneC =
             ConstantInt::getSigned(Type::getInt32Ty(Ctx), ExtrLane);
-        // This may return a Constant if Elm is a Constant.
         auto *ExtrI =
             ExtractElementInst::create(Elm, ExtrLaneC, WhereIt, Ctx, "VPack");
         if (!isa<Constant>(ExtrI))
           WhereIt = std::next(cast<Instruction>(ExtrI)->getIterator());
         Constant *InsertLaneC =
             ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
-        // This may also return a Constant if ExtrI is a Constant.
-        auto *InsertI = InsertElementInst::create(
-            LastInsert, ExtrI, InsertLaneC, WhereIt, Ctx, "VPack");
-        LastInsert = InsertI;
-        if (!isa<Constant>(InsertI))
+        LastInsert = InsertElementInst::create(LastInsert, ExtrI, InsertLaneC,
+                                               WhereIt, Ctx, "VPack");
+        if (!isa<Constant>(LastInsert))
           WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
       }
     } else {
       Constant *InsertLaneC =
           ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
-      // This may be folded into a Constant if LastInsert is a Constant. In
-      // that case we only collect the last constant.
       LastInsert = InsertElementInst::create(LastInsert, Elm, InsertLaneC,
-                                             WhereIt, Ctx, "Pack");
-      if (auto *NewI = dyn_cast<Instruction>(LastInsert))
-        WhereIt = std::next(NewI->getIterator());
+                                             WhereIt, Ctx, "VPack");
+      if (!isa<Constant>(LastInsert))
+        WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
     }
   }
   return LastInsert;
 }
 
-void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
+void BundleVecPass::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
   for (Value *V : Bndl)
     DeadInstrCandidates.insert(cast<Instruction>(V));
-  // Also collect the GEPs of vectorized loads and stores.
   auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
   switch (Opcode) {
   case Instruction::Opcode::Load: {
@@ -278,16 +223,122 @@ void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
   }
 }
 
-Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
-                                  ArrayRef<Value *> UserBndl, unsigned Depth,
-                                  LegalityAnalysis &Legality) {
-  bool StopForDebug =
-      DebugBndlCnt++ >= StopBundle && StopBundle != StopBundleDisabled;
+void BundleVecPass::tryEraseDeadInstrs(StringRef PassName) {
+  DenseMap<BasicBlock *, SmallVector<Instruction *>> SortedDeadInstrCandidates;
+  for (auto *DeadI : DeadInstrCandidates)
+    SortedDeadInstrCandidates[DeadI->getParent()].push_back(DeadI);
+  for (auto &Pair : SortedDeadInstrCandidates)
+    sort(Pair.second,
+         [](Instruction *I1, Instruction *I2) { return I1->comesBefore(I2); });
+  for (const auto &Pair : SortedDeadInstrCandidates) {
+    for (Instruction *I : reverse(Pair.second)) {
+      if (I->hasNUses(0)) {
+        LLVM_DEBUG(dbgs() << PassName << "Erase dead: " << *I << "\n");
+        I->eraseFromParent();
+      }
+    }
+  }
+  DeadInstrCandidates.clear();
+}
+
+void BundleVecPass::emitUnpacksForExternalUses(ArrayRef<Value *> Bndl,
+                                               Value *Vec) {
+  BasicBlock::iterator WhereIt;
+  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
+    WhereIt = std::next(VecI->getIterator());
+  } else {
+    // If Vec is a constant then it should be safe to emit the unpacks at the
+    // top of the block.
+    // Note: Extracts from constants are usually folded to constants.
+    assert(isa<Constant>(Vec) && "Expected constant!");
+    assert(isa<Instruction>(Bndl[0]) &&
+           "A widened Bndl should contain instrs!");
+    BasicBlock *BB = cast<Instruction>(Bndl[0])->getParent();
+    WhereIt =
+        BB->empty()
+            ? BB->begin()
+            : std::next(
+                  VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
+  }
+
+  for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
+    for (User *U : Elm->users()) {
+      // Skip users that we just vectorized.
+      if (IMaps->isVectorized(U))
+        continue;
+      auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
+      Elm->replaceAllUsesWith(LastUnpackV);
+    }
+  }
+}
+
+Action *BundleVecPass::vectorizeRec(ArrayRef<Value *> Bndl,
+                                    ArrayRef<Value *> UserBndl, unsigned Depth,
+                                    LegalityAnalysis &Legality) {
   LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
              VecUtils::dump(Bndl));
-  const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
-                                         : Legality.canVectorize(Bndl);
+
+  const LegalityResult &LegalityRes = [&]() -> const LegalityResult & {
+    if (Direction == VecDirection::BottomUp) {
+      bool StopForDebug = DebugBndlCnt++ >= BottomUpStopBundle &&
+                          BottomUpStopBundle != StopBundleDisabled;
+      if (StopForDebug)
+        return Legality.getForcedPackForDebugging();
+      return Legality.canVectorize(Bndl);
+    }
+    return Legality.canVectorize(Bndl, /*SkipScheduling=*/true);
+  }();
+
   LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
+
+  if (Direction == VecDirection::TopDown) {
+    auto ActionPtr = std::make_unique<Action>(&LegalityRes, Bndl,
+                                              ArrayRef<Value *>(), Depth);
+
+    if (LegalityRes.getSubclassID() == LegalityResultID::Widen)
+      IMaps->registerVector(Bndl, ActionPtr.get());
+
+    // Pre-order push so defs are before uses.
+    Action *Action = ActionPtr.get();
+    Actions.push_back(std::move(ActionPtr));
+
+    if (LegalityRes.getSubclassID() == LegalityResultID::Widen) {
+      // Find users of Bndl to recurse down.
+      // Group the first user of each element if they match.
+      SmallVector<Value *, 4> NextUserBndl;
+      bool CanFormUserBndl = true;
+      for (Value *V : Bndl) {
+        if (V->user_begin() == V->user_end()) {
+          CanFormUserBndl = false;
+          break;
+        }
+        NextUserBndl.push_back(*V->user_begin());
+      }
+
+      if (CanFormUserBndl) {
+        auto *U0 = dyn_cast<Instruction>(NextUserBndl[0]);
+        if (!U0 || IMaps->isVectorized(U0))
+          CanFormUserBndl = false;
+        else {
+          for (Value *U : drop_begin(NextUserBndl)) {
+            auto *UI = dyn_cast<Instruction>(U);
+            if (!UI || UI->getOpcode() != U0->getOpcode() ||
+                UI->getType() != U0->getType() || IMaps->isVectorized(UI)) {
+              CanFormUserBndl = false;
+              break;
+            }
+          }
+        }
+      }
+
+      if (CanFormUserBndl)
+        vectorizeRec(NextUserBndl, {}, Depth + 1, Legality);
+    }
+
+    return Action;
+  }
+
+  // Bottom up direction
   auto ActionPtr =
       std::make_unique<Action>(&LegalityRes, Bndl, UserBndl, Depth);
   SmallVector<Action *> Operands;
@@ -330,87 +381,64 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
   return Action;
 }
 
-#ifndef NDEBUG
-void BottomUpVec::ActionsVector::print(raw_ostream &OS) const {
-  for (auto [Idx, Action] : enumerate(Actions)) {
-    Action->print(OS);
-    OS << "\n";
-  }
-}
-void BottomUpVec::ActionsVector::dump() const { print(dbgs()); }
-#endif // NDEBUG
-
-void BottomUpVec::emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl,
-                                             Value *Vec) {
-  // Find where we should emit the unpacks.
-  BasicBlock::iterator WhereIt;
-  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
-    WhereIt = std::next(VecI->getIterator());
-  } else {
-    // If Vec is a constant then it should be safe to emit the unpacks at the
-    // top of the block.
-    // Note: Extracts from constants are usually folded to constants.
-    assert(isa<Constant>(Vec) && "Expected constant!");
-    assert(isa<Instruction>(Bndl[0]) &&
-           "A widened Bndl should contain instrs!");
-    BasicBlock *BB = cast<Instruction>(Bndl[0])->getParent();
-    WhereIt =
-        BB->empty()
-            ? BB->begin()
-            : std::next(
-                  VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
-  }
-
-  for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
-    for (User *U : Elm->users()) {
-      // Skip users that we just vectorized.
-      if (IMaps->isVectorized(U))
-        continue;
-      auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
-      Elm->replaceAllUsesWith(LastUnpackV);
-    }
-  }
-}
-
-Value *BottomUpVec::emitVectors() {
+Value *BundleVecPass::emitVectors() {
   Value *NewVec = nullptr;
   for (const auto &ActionPtr : Actions) {
     ArrayRef<Value *> Bndl = ActionPtr->Bndl;
     ArrayRef<Value *> UserBndl = ActionPtr->UserBndl;
     const LegalityResult &LegalityRes = *ActionPtr->LegalityRes;
     unsigned Depth = ActionPtr->Depth;
-    auto *UserBB = !UserBndl.empty()
-                       ? cast<Instruction>(UserBndl.front())->getParent()
-                       : cast<Instruction>(Bndl[0])->getParent();
+    BasicBlock *UserBB = !UserBndl.empty()
+                             ? cast<Instruction>(UserBndl[0])->getParent()
+                             : cast<Instruction>(Bndl[0])->getParent();
 
     switch (LegalityRes.getSubclassID()) {
     case LegalityResultID::Widen: {
       auto *I = cast<Instruction>(Bndl[0]);
       SmallVector<Value *, 2> VecOperands;
-      switch (I->getOpcode()) {
-      case Instruction::Opcode::Load:
-        VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
-        break;
-      case Instruction::Opcode::Store: {
-        VecOperands.push_back(ActionPtr->Operands[0]->Vec);
-        VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
-        break;
-      }
-      default:
-        // Visit all operands.
-        for (Action *OpA : ActionPtr->Operands) {
-          auto *VecOp = OpA->Vec;
-          VecOperands.push_back(VecOp);
+      if (Direction == VecDirection::TopDown) {
+        switch (I->getOpcode()) {
+        case Instruction::Opcode::Load:
+          VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+          break;
+        case Instruction::Opcode::Store: {
+          auto OpBndl = getOperand(Bndl, 0);
+          if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+            VecOperands.push_back(OpA->Vec);
+          else
+            VecOperands.push_back(createPack(OpBndl, UserBB));
+          VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+          break;
+        }
+        default:
+          for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) {
+            SmallVector<Value *, 4> OpBndl = getOperand(Bndl, OpIdx);
+            if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+              VecOperands.push_back(OpA->Vec);
+            else
+              VecOperands.push_back(createPack(OpBndl, UserBB));
+          }
+          break;
+        }
+      } else {
+        switch (I->getOpcode()) {
+        case Instruction::Opcode::Load:
+          VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+          break;
+        case Instruction::Opcode::Store:
+          VecOperands.push_back(ActionPtr->Operands[0]->Vec);
+          VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+          break;
+        default:
+          for (Action *OpA : ActionPtr->Operands)
+            VecOperands.push_back(OpA->Vec);
+          break;
         }
-        break;
       }
       NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
-      // Collect any potentially dead scalar instructions, including the
-      // original scalars and pointer operands of loads/stores.
+      LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewVec << "\n");
       if (NewVec != nullptr)
         collectPotentiallyDeadInstrs(Bndl);
-
-      // Emit unpacks for all external uses, if any.
       emitUnpacksForExternalUses(ActionPtr->Bndl, NewVec);
       break;
     }
@@ -423,8 +451,9 @@ Value *BottomUpVec::emitVectors() {
       const ShuffleMask &Mask =
           cast<DiamondReuseWithShuffle>(LegalityRes).getMask();
       NewVec = createShuffle(VecOp, Mask, UserBB);
-      assert(NewVec->getType() == VecOp->getType() &&
-             "Expected same type! Bad mask ?");
+      if (Direction == VecDirection::BottomUp)
+        assert(NewVec->getType() == VecOp->getType() &&
+               "Expected same type! Bad mask ?");
       break;
     }
     case LegalityResultID::DiamondReuseMultiInput: {
@@ -432,13 +461,12 @@ Value *BottomUpVec::emitVectors() {
           cast<DiamondReuseMultiInput>(LegalityRes).getCollectDescr();
       Type *ResTy = VecUtils::getWideType(Bndl[0]->getType(), Bndl.size());
 
-      // TODO: Try to get WhereIt without creating a vector.
       SmallVector<Value *, 4> DescrInstrs;
       for (const auto &ElmDescr : Descr.getDescrs()) {
         auto *V = ElmDescr.needsExtract() ? ElmDescr.getValue()->Vec
                                           : ElmDescr.getScalar();
-        if (auto *I = dyn_cast<Instruction>(V))
-          DescrInstrs.push_back(I);
+        if (auto *Inst = dyn_cast<Instruction>(V))
+          DescrInstrs.push_back(Inst);
       }
       BasicBlock::iterator WhereIt =
           getInsertPointAfterInstrs(DescrInstrs, UserBB);
@@ -447,10 +475,9 @@ Value *BottomUpVec::emitVectors() {
       Context &Ctx = LastV->getContext();
       unsigned Lane = 0;
       for (const auto &ElmDescr : Descr.getDescrs()) {
-        Value *VecOp = nullptr;
         Value *ValueToInsert;
         if (ElmDescr.needsExtract()) {
-          VecOp = ElmDescr.getValue()->Vec;
+          Value *VecOp = ElmDescr.getValue()->Vec;
           ConstantInt *IdxC =
               ConstantInt::get(Type::getInt32Ty(Ctx), ElmDescr.getExtractIdx());
           ValueToInsert = ExtractElementInst::create(
@@ -460,19 +487,10 @@ Value *BottomUpVec::emitVectors() {
         }
         auto NumLanesToInsert = VecUtils::getNumLanes(ValueToInsert);
         if (NumLanesToInsert == 1) {
-          // If we are inserting a scalar element then we need a single insert.
-          //   %VIns = insert %DstVec,  %SrcScalar, Lane
           ConstantInt *LaneC = ConstantInt::get(Type::getInt32Ty(Ctx), Lane);
           LastV = InsertElementInst::create(LastV, ValueToInsert, LaneC,
                                             WhereIt, Ctx, "VIns");
         } else {
-          // If we are inserting a vector element then we need to extract and
-          // insert each vector element one by one with a chain of extracts and
-          // inserts, for example:
-          //   %VExt0 = extract %SrcVec, 0
-          //   %VIns0 = insert  %DstVec, %Vect0, Lane + 0
-          //   %VExt1 = extract %SrcVec, 1
-          //   %VIns1 = insert  %VIns0,  %Vect0, Lane + 1
           for (unsigned LnCnt = 0; LnCnt != NumLanesToInsert; ++LnCnt) {
             auto *ExtrIdxC = ConstantInt::get(Type::getInt32Ty(Ctx), LnCnt);
             auto *ExtrI = ExtractElementInst::create(ValueToInsert, ExtrIdxC,
@@ -489,7 +507,6 @@ Value *BottomUpVec::emitVectors() {
       break;
     }
     case LegalityResultID::Pack: {
-      // If we can't vectorize the seeds then just return.
       if (Depth == 0)
         return nullptr;
       NewVec = createPack(Bndl, UserBB);
@@ -502,11 +519,9 @@ Value *BottomUpVec::emitVectors() {
     }
 #ifndef NDEBUG
     if (AlwaysVerify) {
-      // This helps find broken IR by constantly verifying the function. Note
-      // that this is very expensive and should only be used for debugging.
-      Instruction *I0 = isa<Instruction>(Bndl[0])
-                            ? cast<Instruction>(Bndl[0])
-                            : cast<Instruction>(UserBndl[0]);
+      Instruction *I0 =
+          (isa<Instruction>(Bndl[0]) ? cast<Instruction>(Bndl[0])
+                                     : cast<Instruction>(UserBndl[0]));
       assert(!Utils::verifyFunction(I0->getParent()->getParent(), dbgs()) &&
              "Broken function!");
     }
@@ -515,40 +530,42 @@ Value *BottomUpVec::emitVectors() {
   return NewVec;
 }
 
-bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl,
-                               LegalityAnalysis &Legality) {
+bool BundleVecPass::shouldSkipVectorization() {
+  return InvocationCnt++ >= StopAt && StopAt != StopAtDisabled;
+}
+
+bool BundleVecPass::tryVectorize(ArrayRef<Value *> Bndl,
+                                 LegalityAnalysis &Legality) {
   Change = false;
-  if (LLVM_UNLIKELY(BottomUpInvocationCnt++ >= StopAt &&
-                    StopAt != StopAtDisabled))
+  if (LLVM_UNLIKELY(shouldSkipVectorization()))
     return false;
   DeadInstrCandidates.clear();
   Legality.clear();
   Actions.clear();
   DebugBndlCnt = 0;
   vectorizeRec(Bndl, {}, /*Depth=*/0, Legality);
-  LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "BottomUpVec: Vectorization Actions:\n";
+  LLVM_DEBUG(dbgs() << DEBUG_PREFIX
+                    << (Direction == VecDirection::TopDown ? "TopDownVec"
+                                                           : "BottomUpVec")
+                    << ": Vectorization Actions:\n";
              Actions.dump());
   emitVectors();
-  tryEraseDeadInstrs();
+  tryEraseDeadInstrs(DEBUG_PREFIX);
   return Change;
 }
 
-bool BottomUpVec::runOnRegion(Region &Rgn, const Analyses &A) {
+bool BundleVecPass::runOnRegion(Region &Rgn, const Analyses &A) {
   const auto &SeedSlice = Rgn.getAux();
   assert(SeedSlice.size() >= 2 && "Bad slice!");
+
   Function &F = *SeedSlice[0]->getParent()->getParent();
   IMaps = std::make_unique<InstrMaps>();
   LegalityAnalysis Legality(A.getAA(), A.getScalarEvolution(),
                             F.getParent()->getDataLayout(), F.getContext(),
                             *IMaps);
 
-  // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
   SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
-  // Try to vectorize starting from the seed slice. The returned value
-  // is true if we found vectorizable code and generated some vector
-  // code for it. It does not mean that the code is profitable.
   return tryVectorize(SeedSliceVals, Legality);
 }
 
-} // namespace sandboxir
-} // namespace llvm
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index 10ba595910ee9..40d03c3886c0b 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -26,6 +26,7 @@ REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
 REGION_PASS("tr-revert", ::llvm::sandboxir::TransactionAlwaysRevert)
 REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
 REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
+REGION_PASS("top-down-vec", ::llvm::sandboxir::TopDownVec)
 REGION_PASS("load-store-vec", ::llvm::sandboxir::LoadStoreVec)
 
 #undef REGION_PASS
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 9326869461082..e53895660859c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -9,6 +9,7 @@
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysRevert.h"
diff --git a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
index 593965ab01680..624cf3d918362 100644
--- a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
 ; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
 
 ; Checks the handling of users outside the vectorized graph.
 
@@ -13,6 +14,20 @@ define void @external_users(ptr %ptr) {
 ; CHECK-NEXT:    store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
 ; CHECK-NEXT:    [[USER:%.*]] = fneg float [[SUB0]]
 ; CHECK-NEXT:    ret void
+;
+; TOPDOWN-LABEL: define void @external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT:    [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT:    [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT:    [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
+; TOPDOWN-NEXT:    [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT:    [[SUB1:%.*]] = fsub float [[LD1]], 0.000000e+00
+; TOPDOWN-NEXT:    [[VPACK:%.*]] = insertelement <2 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT:    [[VPACK1:%.*]] = insertelement <2 x float> [[VPACK]], float [[SUB1]], i32 1, !sandboxvec [[META0]]
+; TOPDOWN-NEXT:    store <2 x float> [[VPACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
+; TOPDOWN-NEXT:    [[USER:%.*]] = fneg float [[SUB0]]
+; TOPDOWN-NEXT:    ret void
 ;
   %ptr0 = getelementptr float, ptr %ptr, i32 0
   %ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -35,6 +50,17 @@ define void @external_user_of_constant(ptr %ptr, ptr %ptrX) {
 ; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
 ; CHECK-NEXT:    store i32 0, ptr [[PTRX]], align 4
 ; CHECK-NEXT:    ret void
+;
+; TOPDOWN-LABEL: define void @external_user_of_constant(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], ptr [[PTRX:%.*]]) {
+; TOPDOWN-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT:    [[ZEXT0:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT:    [[ZEXT1:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT:    [[VPACK:%.*]] = insertelement <2 x i32> poison, i32 [[ZEXT0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT:    [[VPACK1:%.*]] = insertelement <2 x i32> [[VPACK]], i32 [[ZEXT1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT:    store <2 x i32> [[VPACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META1]]
+; TOPDOWN-NEXT:    store i32 [[ZEXT0]], ptr [[PTRX]], align 4
+; TOPDOWN-NEXT:    ret void
 ;
   %ptr0 = getelementptr float, ptr %ptr, i32 0
   %ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -57,6 +83,23 @@ define void @vector_external_users(ptr %ptr) {
 ; CHECK-NEXT:    store <3 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
 ; CHECK-NEXT:    [[USER:%.*]] = fneg <2 x float> [[UNPACKINS2]]
 ; CHECK-NEXT:    ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT:    [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT:    [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT:    [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT:    [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT:    [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT:    [[VPACK:%.*]] = insertelement <3 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT:    [[VPACK1:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META2]]
+; TOPDOWN-NEXT:    [[VPACK2:%.*]] = insertelement <3 x float> [[VPACK]], float [[VPACK1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT:    [[VPACK3:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT:    [[VPACK4:%.*]] = insertelement <3 x float> [[VPACK2]], float [[VPACK3]], i32 2, !sandboxvec [[META2]]
+; TOPDOWN-NEXT:    store <3 x float> [[VPACK4]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
+; TOPDOWN-NEXT:    [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT:    ret void
 ;
   %ptr0 = getelementptr float, ptr %ptr, i32 0
   %ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -80,6 +123,26 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
 ; CHECK-NEXT:    store <4 x float> [[VEC]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
 ; CHECK-NEXT:    [[USER:%.*]] = fneg <2 x float> [[UNPACK]]
 ; CHECK-NEXT:    ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users_lane_and_index_differ(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT:    [[PTR0:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT:    [[PTR1:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT:    [[LD0:%.*]] = load <2 x float>, ptr [[PTR0]], align 8
+; TOPDOWN-NEXT:    [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT:    [[SUB0:%.*]] = fsub <2 x float> [[LD0]], zeroinitializer
+; TOPDOWN-NEXT:    [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT:    [[VPACK:%.*]] = extractelement <2 x float> [[SUB0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT:    [[VPACK1:%.*]] = insertelement <4 x float> poison, float [[VPACK]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[VPACK2:%.*]] = extractelement <2 x float> [[SUB0]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[VPACK3:%.*]] = insertelement <4 x float> [[VPACK1]], float [[VPACK2]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[VPACK4:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[VPACK5:%.*]] = insertelement <4 x float> [[VPACK3]], float [[VPACK4]], i32 2, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[VPACK6:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[VPACK7:%.*]] = insertelement <4 x float> [[VPACK5]], float [[VPACK6]], i32 3, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    store <4 x float> [[VPACK7]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT:    ret void
 ;
   %ptr0 = getelementptr <2 x float>, ptr %ptr, i32 0
   %ptr1 = getelementptr <2 x float>, ptr %ptr, i32 1
@@ -99,3 +162,8 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
 ; CHECK: [[META2]] = distinct !{!"sandboxregion"}
 ; CHECK: [[META3]] = distinct !{!"sandboxregion"}
 ;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index 743d705fd48ff..2a50ba84e53db 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
 
 define void @pack_constants(ptr %ptr) {
 ; CHECK-LABEL: define void @pack_constants(
@@ -7,6 +8,12 @@ define void @pack_constants(ptr %ptr) {
 ; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
 ; CHECK-NEXT:    store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret void
+;
+; TOPDOWN-LABEL: define void @pack_constants(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT:    store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT:    ret void
 ;
   %ptr0 = getelementptr i8, ptr %ptr, i32 0
   %ptr1 = getelementptr i8, ptr %ptr, i32 1
@@ -27,14 +34,31 @@ define void @packPHIs(ptr %ptr) {
 ; CHECK-NEXT:    [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
 ; CHECK-NEXT:    [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
 ; CHECK-NEXT:    [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
-; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; CHECK-NEXT:    [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT:    [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
 ; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT:    store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; CHECK-NEXT:    store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
 ; CHECK-NEXT:    br label %[[LOOP]]
 ; CHECK:       [[EXIT:.*:]]
 ; CHECK-NEXT:    ret void
 ;
+; TOPDOWN-LABEL: define void @packPHIs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT:  [[ENTRY:.*]]:
+; TOPDOWN-NEXT:    br label %[[LOOP:.*]]
+; TOPDOWN:       [[LOOP]]:
+; TOPDOWN-NEXT:    [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT:    [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT:    [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT:    [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT:    [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT:    [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT:    store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT:    br label %[[LOOP]]
+; TOPDOWN:       [[EXIT:.*:]]
+; TOPDOWN-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -63,14 +87,31 @@ define void @packFromOtherBB(ptr %ptr, i8 %val) {
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
 ; CHECK-NEXT:    [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
-; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT:    [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT:    [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
 ; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT:    store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; CHECK-NEXT:    store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
 ; CHECK-NEXT:    br label %[[LOOP]]
 ; CHECK:       [[EXIT:.*:]]
 ; CHECK-NEXT:    ret void
 ;
+; TOPDOWN-LABEL: define void @packFromOtherBB(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[VAL:%.*]]) {
+; TOPDOWN-NEXT:  [[ENTRY:.*]]:
+; TOPDOWN-NEXT:    [[ADD0:%.*]] = add i8 [[VAL]], 0
+; TOPDOWN-NEXT:    [[MUL1:%.*]] = mul i8 [[VAL]], 1
+; TOPDOWN-NEXT:    br label %[[LOOP:.*]]
+; TOPDOWN:       [[LOOP]]:
+; TOPDOWN-NEXT:    [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT:    [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT:    [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT:    [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT:    store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT:    br label %[[LOOP]]
+; TOPDOWN:       [[EXIT:.*:]]
+; TOPDOWN-NEXT:    ret void
+;
 entry:
   %add0 = add i8 %val, 0
   %mul1 = mul i8 %val, 1
@@ -97,12 +138,25 @@ define void @packFromDiffBBs(ptr %ptr, i8 %v) {
 ; CHECK-NEXT:    br label %[[BB:.*]]
 ; CHECK:       [[BB]]:
 ; CHECK-NEXT:    [[ADD1:%.*]] = add i8 [[V]], 2
-; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
-; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; CHECK-NEXT:    [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT:    [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
 ; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT:    store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; CHECK-NEXT:    store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
 ; CHECK-NEXT:    ret void
 ;
+; TOPDOWN-LABEL: define void @packFromDiffBBs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[V:%.*]]) {
+; TOPDOWN-NEXT:  [[ENTRY:.*:]]
+; TOPDOWN-NEXT:    [[ADD0:%.*]] = add i8 [[V]], 1
+; TOPDOWN-NEXT:    br label %[[BB:.*]]
+; TOPDOWN:       [[BB]]:
+; TOPDOWN-NEXT:    [[ADD1:%.*]] = add i8 [[V]], 2
+; TOPDOWN-NEXT:    [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT:    [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT:    store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT:    ret void
+;
 entry:
   %add0 = add i8 %v, 1
   br label %bb
@@ -121,3 +175,8 @@ bb:
 ; CHECK: [[META2]] = distinct !{!"sandboxregion"}
 ; CHECK: [[META3]] = distinct !{!"sandboxregion"}
 ;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
new file mode 100644
index 0000000000000..a9db7d9225685
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 \
+; RUN:    -sbvec-collect-seeds=loads \
+; RUN:    -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" \
+; RUN:    %s -S | FileCheck %s
+
+define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_fadd_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META0]]
+; CHECK-NEXT:    [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT:    store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META0]]
+; CHECK-NEXT:    ret void
+;
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  %ld0 = load float, ptr %ptr0
+  %ld1 = load float, ptr %ptr1
+
+  %fadd0 = fadd float %ld0, %ld0
+  %fadd1 = fadd float %ld1, %ld1
+
+  %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+  %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+  store float %fadd0, ptr %ptr2_0
+  store float %fadd1, ptr %ptr2_1
+  ret void
+}
+
+define void @load_chain_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_chain_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT:    [[VEC:%.*]] = fmul <2 x float> [[VECL]], splat (float 3.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT:    [[VEC1:%.*]] = fadd <2 x float> [[VEC]], splat (float 2.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT:    [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT:    store <2 x float> [[VEC1]], ptr [[PTR2_0]], align 4, !sandboxvec [[META1]]
+; CHECK-NEXT:    ret void
+;
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  %ld0 = load float, ptr %ptr0
+  %ld1 = load float, ptr %ptr1
+
+  %fmul0 = fmul float %ld0, 3.0
+  %fmul1 = fmul float %ld1, 3.0
+
+  %fadd0 = fadd float %fmul0, 2.0
+  %fadd1 = fadd float %fmul1, 2.0
+
+  %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+  %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+  store float %fadd0, ptr %ptr2_0
+  store float %fadd1, ptr %ptr2_1
+  ret void
+}
+
+define float @load_fadd_external_use(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define float @load_fadd_external_use(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META2]]
+; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META2]]
+; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VEC]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT:    [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT:    [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT:    store float [[UNPACK]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT:    store float [[UNPACK1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT:    ret float [[UNPACK]]
+;
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  %ld0 = load float, ptr %ptr0
+  %ld1 = load float, ptr %ptr1
+
+  %fadd0 = fadd float %ld0, %ld0
+  %fadd1 = fadd float %ld1, %ld1
+
+  %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+  %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+  store float %fadd0, ptr %ptr2_0
+  store float %fadd1, ptr %ptr2_1
+
+  ret float %fadd0
+}
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+; CHECK: [[META2]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index e8f04be8eb71d..613aac0501282 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -18,7 +18,7 @@ static_library("Vectorize") {
     "SandboxVectorizer/InstrMaps.cpp",
     "SandboxVectorizer/Interval.cpp",
     "SandboxVectorizer/Legality.cpp",
-    "SandboxVectorizer/Passes/BottomUpVec.cpp",
+    "SandboxVectorizer/Passes/BundleVecPass.cpp",
     "SandboxVectorizer/Passes/LoadStoreVec.cpp",
     "SandboxVectorizer/Passes/PackReuse.cpp",
     "SandboxVectorizer/Passes/RegionsFromBBs.cpp",



More information about the llvm-commits mailing list