[llvm] [SandboxVectorizer] Implement topdown vectorizer (PR #205249)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 22 20:54:24 PDT 2026
https://github.com/gandhi56 created https://github.com/llvm/llvm-project/pull/205249
This patch introduces the `top-down-vec` pass to the Sandbox Vectorizer, adding the ability to traverse use-def chains top-down to discover and collect vectorization opportunities.
Key changes include:
* TopDownVec Pass: Implemented `TopDownVec` which recursively processes value bundles top-down, creates vectorization actions (widening, packing, shuffles), and emits the final vector IR.
* Shared Infrastructure (VecPassBase): Extracted common IR emission logic out of `BottomUpVec` and into a new shared base class, `VecPassBase`. Functions for generating vector instructions, handling diamond reuse, creating shuffles/packs, and collecting dead instructions are now shared between the bottom-up and top-down vectorizers to prevent code duplication.
* Pass Registration: Exposed `top-down-vec` in `PassRegistry.def` and `SandboxVectorizerPassBuilder`, allowing it to be invoked within pass pipelines via `opt`.
* Testing: Added comprehensive lit tests (such as `topdown_vec.ll`, `pack.ll`, and `external_uses.ll`) to verify correct top-down traversal, external use unpacking, and proper fallback to scalar packing.
>From 4150a786d38a6cbb0383bd8fcedb9d5d17a2cc76 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Sun, 14 Jun 2026 17:01:48 -0400
Subject: [PATCH] [SandboxVectorizer] Implement topdown vectorizer
This patch introduces the `top-down-vec` pass to the Sandbox Vectorizer,
adding the ability to traverse use-def chains top-down to discover and
collect vectorization opportunities.
Key changes include:
* TopDownVec Pass: Implemented `TopDownVec` which recursively processes
value bundles top-down, creates vectorization actions (widening, packing,
shuffles), and emits the final vector IR.
* Shared Infrastructure (VecPassBase): Extracted common IR emission logic
out of `BottomUpVec` and into a new shared base class, `VecPassBase`.
Functions for generating vector instructions, handling diamond reuse,
creating shuffles/packs, and collecting dead instructions are now shared
between the bottom-up and top-down vectorizers to prevent code
duplication.
* Pass Registration: Exposed `top-down-vec` in `PassRegistry.def` and
`SandboxVectorizerPassBuilder`, allowing it to be invoked within pass
pipelines via `opt`.
* Testing: Added comprehensive lit tests (such as `topdown_vec.ll`,
`pack.ll`, and `external_uses.ll`) to verify correct top-down traversal,
external use unpacking, and proper fallback to scalar packing.
---
.../SandboxVectorizer/Passes/BottomUpVec.h | 58 +---
.../SandboxVectorizer/Passes/TopDownVec.h | 37 +++
.../SandboxVectorizer/Passes/VecPassBase.h | 121 ++++++++
.../Vectorize/SandboxVectorizer/VecUtils.h | 8 +-
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 2 +
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 283 +-----------------
.../SandboxVectorizer/Passes/PassRegistry.def | 1 +
.../SandboxVectorizer/Passes/TopDownVec.cpp | 262 ++++++++++++++++
.../SandboxVectorizer/Passes/VecPassBase.cpp | 257 ++++++++++++++++
.../SandboxVectorizerPassBuilder.cpp | 1 +
.../SandboxVectorizer/external_uses.ll | 68 +++++
.../test/Transforms/SandboxVectorizer/pack.ll | 77 ++++-
.../SandboxVectorizer/topdown_vec.ll | 94 ++++++
13 files changed, 923 insertions(+), 346 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.h
create mode 100644 llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.cpp
create mode 100644 llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.cpp
create mode 100644 llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index d024bb74ac0c9..83db0ec237195 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -15,10 +15,8 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/SandboxIR/Constant.h"
-#include "llvm/SandboxIR/Pass.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.h"
namespace llvm::sandboxir {
@@ -32,65 +30,15 @@ namespace llvm::sandboxir {
/// profitable or not. For now profitability is checked at the end of the region
/// pass pipeline by a dedicated pass that accepts or rejects the IR
/// transaction, depending on the cost.
-class LLVM_ABI BottomUpVec final : public RegionPass {
- bool Change = false;
- /// The original instructions that are potentially dead after vectorization.
- DenseSet<Instruction *> DeadInstrCandidates;
- /// Maps scalars to vectors.
- std::unique_ptr<InstrMaps> IMaps;
+class LLVM_ABI BottomUpVec final : public VecPassBase {
/// Counter used for force-stopping the vectorizer after this many
/// invocations. Used for debugging miscompiles.
unsigned long BottomUpInvocationCnt = 0;
- /// Creates and returns a vector instruction that replaces the instructions in
- /// \p Bndl. \p Operands are the already vectorized operands.
- Value *createVectorInstr(ArrayRef<Value *> Bndl, ArrayRef<Value *> Operands);
- /// Erases all dead instructions from the dead instruction candidates
- /// collected during vectorization.
- void tryEraseDeadInstrs();
- /// Creates a shuffle instruction that shuffles \p VecOp according to \p Mask.
- /// \p UserBB is the block of the user bundle.
- Value *createShuffle(Value *VecOp, const ShuffleMask &Mask,
- BasicBlock *UserBB);
- /// Packs all elements of \p ToPack into a vector and returns that vector. \p
- /// UserBB is the block of the user bundle.
- Value *createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB);
- /// After we create vectors for groups of instructions, the original
- /// instructions are potentially dead and may need to be removed. This
- /// function helps collect these instructions (along with the pointer operands
- /// for loads/stores) so that they can be cleaned up later.
- void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
-
- /// Helper class describing how(if) to vectorize the code.
- class ActionsVector {
- private:
- SmallVector<std::unique_ptr<Action>, 16> Actions;
-
- public:
- auto begin() const { return Actions.begin(); }
- auto end() const { return Actions.end(); }
- void push_back(std::unique_ptr<Action> &&ActPtr) {
- ActPtr->Idx = Actions.size();
- Actions.push_back(std::move(ActPtr));
- }
- void clear() { Actions.clear(); }
-#ifndef NDEBUG
- void print(raw_ostream &OS) const;
- void dump() const;
-#endif // NDEBUG
- };
- ActionsVector Actions;
- /// Helper counter for debugging. It counts the bundles that we attempt to
- /// vectorize in vectorizeRec().
- unsigned DebugBndlCnt = 0;
-
/// Recursively try to vectorize \p Bndl and its operands. This populates the
/// `Actions` vector.
Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
unsigned Depth, LegalityAnalysis &Legality);
- /// If the values in \p Bndl have external users, then emit unpacks and
- /// connect them to the users. \p Vec is the vectorized form of \p Bndl.
- void emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl, Value *Vec);
/// Generate vector instructions based on `Actions` and return the last vector
/// created.
Value *emitVectors();
@@ -98,7 +46,7 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
bool tryVectorize(ArrayRef<Value *> Seeds, LegalityAnalysis &Legality);
public:
- BottomUpVec() : RegionPass("bottom-up-vec") {}
+ BottomUpVec() : VecPassBase("bottom-up-vec") {}
bool runOnRegion(Region &Rgn, const Analyses &A) final;
};
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
new file mode 100644
index 0000000000000..04ab0d7370f5f
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h
@@ -0,0 +1,37 @@
+//===- TopDownVec.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A Top-Down Vectorizer pass.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.h"
+
+namespace llvm::sandboxir {
+
+class LLVM_ABI TopDownVec final : public VecPassBase {
+ unsigned long TopDownInvocationCnt = 0;
+
+ Action *vectorizeRec(ArrayRef<Value *> Bndl, unsigned Depth,
+ LegalityAnalysis &Legality);
+ Value *emitVectors();
+ bool tryVectorize(ArrayRef<Value *> Seeds, LegalityAnalysis &Legality);
+
+public:
+ TopDownVec() : VecPassBase("top-down-vec") {}
+ bool runOnRegion(Region &Rgn, const Analyses &A) final;
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TOPDOWNVEC_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.h
new file mode 100644
index 0000000000000..cc8f441c67c53
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.h
@@ -0,0 +1,121 @@
+//===- VecPassBase.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Abstract base class shared by the top-down and bottom-up vectorizer passes.
+// Houses the common state (InstrMaps, dead-instruction bookkeeping, actions
+// list) and the utility functions used by both vectorizer passes.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_VECPASSBASE_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_VECPASSBASE_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+namespace llvm::sandboxir {
+
+/// Abstract base class shared by TopDownVec and BottomUpVec.
+/// It owns the state common to both vectorizers and implements the utility
+/// functions used by both vectorizer passes.
+class LLVM_ABI VecPassBase : public RegionPass {
+protected:
+ /// Set to true whenever the pass emits vector code in the current region.
+ bool Change = false;
+ /// Scalar instructions that may be dead after vectorization.
+ DenseSet<Instruction *> DeadInstrCandidates;
+ /// Maps scalar values/instructions to their vector counterparts.
+ std::unique_ptr<InstrMaps> IMaps;
+ /// Helper counter for debugging — counts bundles attempted in vectorizeRec.
+ unsigned DebugBndlCnt = 0;
+
+ /// Ordered list of vectorization decisions produced by vectorizeRec.
+ class ActionsVector {
+ SmallVector<std::unique_ptr<Action>, 16> Actions;
+
+ public:
+ auto begin() const { return Actions.begin(); }
+ auto end() const { return Actions.end(); }
+ auto rbegin() const { return Actions.rbegin(); }
+ auto rend() const { return Actions.rend(); }
+ void push_back(std::unique_ptr<Action> &&ActPtr) {
+ ActPtr->Idx = Actions.size();
+ Actions.push_back(std::move(ActPtr));
+ }
+ void clear() { Actions.clear(); }
+#ifndef NDEBUG
+ void print(raw_ostream &OS) const;
+ LLVM_DUMP_METHOD void dump() const;
+#endif // NDEBUG
+ };
+ ActionsVector Actions;
+
+ explicit VecPassBase(StringRef Name) : RegionPass(Name) {}
+
+ // -----------------------------------------------------------------------
+ // Utility functions. These are owned by VecPassBase and accessible only
+ // to subclasses (TopDownVec and BottomUpVec) through inheritance.
+ // -----------------------------------------------------------------------
+
+ /// \Returns the operand at \p OpIdx for each instruction in \p Bndl.
+ static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
+ unsigned OpIdx) {
+ SmallVector<Value *, 4> Operands;
+ for (Value *BndlV : Bndl)
+ Operands.push_back(cast<Instruction>(BndlV)->getOperand(OpIdx));
+ return Operands;
+ }
+
+ /// \Returns the BB iterator after the lowest instruction in \p Vals, or the
+ /// top of BB (after any PHIs) if no instruction is found in \p Vals.
+ static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef<Value *> Vals,
+ BasicBlock *BB) {
+ auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB));
+ if (BotI == nullptr)
+ return BB->empty() ? BB->begin()
+ : std::next(VecUtils::getLastPHIOrSelf(&*BB->begin())
+ ->getIterator());
+ return std::next(BotI->getIterator());
+ }
+
+ /// Creates and returns a new vector instruction widening \p Bndl with
+ /// \p Operands as the vector operands.
+ static Value *createVectorInstr(ArrayRef<Value *> Bndl,
+ ArrayRef<Value *> Operands);
+
+ /// Creates a shuffle of \p VecOp according to \p Mask, inserted in \p UserBB.
+ static Value *createShuffle(Value *VecOp, const ShuffleMask &Mask,
+ BasicBlock *UserBB);
+
+ /// Packs all scalars/vectors in \p ToPack into a single vector, inserted in
+ /// \p UserBB.
+ static Value *createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB);
+
+ /// Adds the instructions in \p Bndl (and pointer operands of loads/stores)
+ /// to DeadInstrCandidates for later cleanup.
+ void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
+
+ /// Erases all zero-use instructions from DeadInstrCandidates.
+ /// \p PassName is used as the debug-output prefix.
+ void tryEraseDeadInstrs(StringRef PassName);
+
+ /// For each element of \p Bndl that has external uses (users not yet
+ /// vectorized), emits extract instructions from \p Vec and replaces the
+ /// scalar uses with them.
+ void emitUnpacksForExternalUses(ArrayRef<Value *> Bndl, Value *Vec);
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_VECPASSBASE_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 99e411a1723b0..283b627879bfd 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -12,6 +12,8 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/SandboxIR/Type.h"
@@ -33,6 +35,10 @@ template <> struct DenseMapInfo<SmallVector<sandboxir::Value *>> {
namespace sandboxir {
+class ShuffleMask;
+class InstrMaps;
+class VecPassBase;
+
class VecUtils {
public:
/// \Returns the number of elements in \p Ty. That is the number of lanes if a
@@ -379,7 +385,7 @@ class VecUtils {
}
#ifndef NDEBUG
- /// Helper dump function for debugging.
+ /// Print \p Bndl. Dump function for debugging.
LLVM_DUMP_METHOD static void dump(ArrayRef<Value *> Bndl);
LLVM_DUMP_METHOD static void dump(ArrayRef<Instruction *> Bndl);
#endif // NDEBUG
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index d081747b821a4..db68fa7cce885 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -9,11 +9,13 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/Interval.cpp
SandboxVectorizer/Legality.cpp
SandboxVectorizer/Passes/BottomUpVec.cpp
+ SandboxVectorizer/Passes/VecPassBase.cpp
SandboxVectorizer/Passes/LoadStoreVec.cpp
SandboxVectorizer/Passes/PackReuse.cpp
SandboxVectorizer/Passes/RegionsFromBBs.cpp
SandboxVectorizer/Passes/RegionsFromMetadata.cpp
SandboxVectorizer/Passes/SeedCollection.cpp
+ SandboxVectorizer/Passes/TopDownVec.cpp
SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
SandboxVectorizer/Passes/TransactionSave.cpp
SandboxVectorizer/RegionWithScore.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 6bf257fcf8b1d..26e2f09d472e1 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -40,244 +40,6 @@ static cl::opt<unsigned long>
namespace sandboxir {
-static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
- unsigned OpIdx) {
- SmallVector<Value *, 4> Operands;
- for (Value *BndlV : Bndl) {
- auto *BndlI = cast<Instruction>(BndlV);
- Operands.push_back(BndlI->getOperand(OpIdx));
- }
- return Operands;
-}
-
-/// \Returns the BB iterator after the lowest instruction in \p Vals, or the top
-/// of BB if no instruction found in \p Vals.
-static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef<Value *> Vals,
- BasicBlock *BB) {
- auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB));
- if (BotI == nullptr)
- // We are using BB->begin() (or after PHIs) as the fallback insert point.
- return BB->empty()
- ? BB->begin()
- : std::next(
- VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
- return std::next(BotI->getIterator());
-}
-
-Value *BottomUpVec::createVectorInstr(ArrayRef<Value *> Bndl,
- ArrayRef<Value *> Operands) {
- auto CreateVectorInstr = [](ArrayRef<Value *> Bndl,
- ArrayRef<Value *> Operands) -> Value * {
- assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
- "Expect Instructions!");
- auto &Ctx = Bndl[0]->getContext();
-
- Type *ScalarTy = VecUtils::getElementType(Utils::getExpectedType(Bndl[0]));
- auto *VecTy = VecUtils::getWideType(ScalarTy, VecUtils::getNumLanes(Bndl));
-
- BasicBlock::iterator WhereIt = getInsertPointAfterInstrs(
- Bndl, cast<Instruction>(Bndl[0])->getParent());
-
- auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
- switch (Opcode) {
- case Instruction::Opcode::ZExt:
- case Instruction::Opcode::SExt:
- case Instruction::Opcode::FPToUI:
- case Instruction::Opcode::FPToSI:
- case Instruction::Opcode::FPExt:
- case Instruction::Opcode::PtrToInt:
- case Instruction::Opcode::IntToPtr:
- case Instruction::Opcode::SIToFP:
- case Instruction::Opcode::UIToFP:
- case Instruction::Opcode::Trunc:
- case Instruction::Opcode::FPTrunc:
- case Instruction::Opcode::BitCast: {
- assert(Operands.size() == 1u && "Casts are unary!");
- return CastInst::create(VecTy, Opcode, Operands[0], WhereIt, Ctx,
- "VCast");
- }
- case Instruction::Opcode::FCmp:
- case Instruction::Opcode::ICmp: {
- auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
- assert(all_of(drop_begin(Bndl),
- [Pred](auto *SBV) {
- return cast<CmpInst>(SBV)->getPredicate() == Pred;
- }) &&
- "Expected same predicate across bundle.");
- return CmpInst::create(Pred, Operands[0], Operands[1], WhereIt, Ctx,
- "VCmp");
- }
- case Instruction::Opcode::Select: {
- return SelectInst::create(Operands[0], Operands[1], Operands[2], WhereIt,
- Ctx, "Vec");
- }
- case Instruction::Opcode::FNeg: {
- auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
- auto OpC = UOp0->getOpcode();
- return UnaryOperator::createWithCopiedFlags(OpC, Operands[0], UOp0,
- WhereIt, Ctx, "Vec");
- }
- case Instruction::Opcode::Add:
- case Instruction::Opcode::FAdd:
- case Instruction::Opcode::Sub:
- case Instruction::Opcode::FSub:
- case Instruction::Opcode::Mul:
- case Instruction::Opcode::FMul:
- case Instruction::Opcode::UDiv:
- case Instruction::Opcode::SDiv:
- case Instruction::Opcode::FDiv:
- case Instruction::Opcode::URem:
- case Instruction::Opcode::SRem:
- case Instruction::Opcode::FRem:
- case Instruction::Opcode::Shl:
- case Instruction::Opcode::LShr:
- case Instruction::Opcode::AShr:
- case Instruction::Opcode::And:
- case Instruction::Opcode::Or:
- case Instruction::Opcode::Xor: {
- auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
- auto *LHS = Operands[0];
- auto *RHS = Operands[1];
- return BinaryOperator::createWithCopiedFlags(
- BinOp0->getOpcode(), LHS, RHS, BinOp0, WhereIt, Ctx, "Vec");
- }
- case Instruction::Opcode::Load: {
- auto *Ld0 = cast<LoadInst>(Bndl[0]);
- Value *Ptr = Ld0->getPointerOperand();
- return LoadInst::create(VecTy, Ptr, Ld0->getAlign(), WhereIt, Ctx,
- "VecL");
- }
- case Instruction::Opcode::Store: {
- auto Align = cast<StoreInst>(Bndl[0])->getAlign();
- Value *Val = Operands[0];
- Value *Ptr = Operands[1];
- return StoreInst::create(Val, Ptr, Align, WhereIt, Ctx);
- }
- case Instruction::Opcode::UncondBr:
- case Instruction::Opcode::CondBr:
- case Instruction::Opcode::Ret:
- case Instruction::Opcode::PHI:
- case Instruction::Opcode::AddrSpaceCast:
- case Instruction::Opcode::Call:
- case Instruction::Opcode::GetElementPtr:
- llvm_unreachable("Unimplemented");
- break;
- default:
- llvm_unreachable("Unimplemented");
- break;
- }
- llvm_unreachable("Missing switch case!");
- // TODO: Propagate debug info.
- };
-
- auto *NewI = CreateVectorInstr(Bndl, Operands);
- LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewI << "\n");
- return NewI;
-}
-
-void BottomUpVec::tryEraseDeadInstrs() {
- DenseMap<BasicBlock *, SmallVector<Instruction *>> SortedDeadInstrCandidates;
- // The dead instrs could span BBs, so we need to collect and sort them per BB.
- for (auto *DeadI : DeadInstrCandidates)
- SortedDeadInstrCandidates[DeadI->getParent()].push_back(DeadI);
- for (auto &Pair : SortedDeadInstrCandidates)
- sort(Pair.second,
- [](Instruction *I1, Instruction *I2) { return I1->comesBefore(I2); });
- for (const auto &Pair : SortedDeadInstrCandidates) {
- for (Instruction *I : reverse(Pair.second)) {
- if (I->hasNUses(0)) {
- // Erase the dead instructions bottom-to-top.
- LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Erase dead: " << *I << "\n");
- I->eraseFromParent();
- }
- }
- }
- DeadInstrCandidates.clear();
-}
-
-Value *BottomUpVec::createShuffle(Value *VecOp, const ShuffleMask &Mask,
- BasicBlock *UserBB) {
- BasicBlock::iterator WhereIt = getInsertPointAfterInstrs({VecOp}, UserBB);
- return ShuffleVectorInst::create(VecOp, VecOp, Mask, WhereIt,
- VecOp->getContext(), "VShuf");
-}
-
-Value *BottomUpVec::createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB) {
- BasicBlock::iterator WhereIt = getInsertPointAfterInstrs(ToPack, UserBB);
-
- Type *ScalarTy = VecUtils::getCommonScalarType(ToPack);
- unsigned Lanes = VecUtils::getNumLanes(ToPack);
- Type *VecTy = VecUtils::getWideType(ScalarTy, Lanes);
-
- // Create a series of pack instructions.
- Value *LastInsert = PoisonValue::get(VecTy);
-
- Context &Ctx = ToPack[0]->getContext();
-
- unsigned InsertIdx = 0;
- for (Value *Elm : ToPack) {
- // An element can be either scalar or vector. We need to generate different
- // IR for each case.
- if (Elm->getType()->isVectorTy()) {
- unsigned NumElms =
- cast<FixedVectorType>(Elm->getType())->getNumElements();
- for (auto ExtrLane : seq<int>(0, NumElms)) {
- // We generate extract-insert pairs, for each lane in `Elm`.
- Constant *ExtrLaneC =
- ConstantInt::getSigned(Type::getInt32Ty(Ctx), ExtrLane);
- // This may return a Constant if Elm is a Constant.
- auto *ExtrI =
- ExtractElementInst::create(Elm, ExtrLaneC, WhereIt, Ctx, "VPack");
- if (!isa<Constant>(ExtrI))
- WhereIt = std::next(cast<Instruction>(ExtrI)->getIterator());
- Constant *InsertLaneC =
- ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
- // This may also return a Constant if ExtrI is a Constant.
- auto *InsertI = InsertElementInst::create(
- LastInsert, ExtrI, InsertLaneC, WhereIt, Ctx, "VPack");
- LastInsert = InsertI;
- if (!isa<Constant>(InsertI))
- WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
- }
- } else {
- Constant *InsertLaneC =
- ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
- // This may be folded into a Constant if LastInsert is a Constant. In
- // that case we only collect the last constant.
- LastInsert = InsertElementInst::create(LastInsert, Elm, InsertLaneC,
- WhereIt, Ctx, "Pack");
- if (auto *NewI = dyn_cast<Instruction>(LastInsert))
- WhereIt = std::next(NewI->getIterator());
- }
- }
- return LastInsert;
-}
-
-void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
- for (Value *V : Bndl)
- DeadInstrCandidates.insert(cast<Instruction>(V));
- // Also collect the GEPs of vectorized loads and stores.
- auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
- switch (Opcode) {
- case Instruction::Opcode::Load: {
- for (Value *V : drop_begin(Bndl))
- if (auto *Ptr =
- dyn_cast<Instruction>(cast<LoadInst>(V)->getPointerOperand()))
- DeadInstrCandidates.insert(Ptr);
- break;
- }
- case Instruction::Opcode::Store: {
- for (Value *V : drop_begin(Bndl))
- if (auto *Ptr =
- dyn_cast<Instruction>(cast<StoreInst>(V)->getPointerOperand()))
- DeadInstrCandidates.insert(Ptr);
- break;
- }
- default:
- break;
- }
-}
-
Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
ArrayRef<Value *> UserBndl, unsigned Depth,
LegalityAnalysis &Legality) {
@@ -330,48 +92,6 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
return Action;
}
-#ifndef NDEBUG
-void BottomUpVec::ActionsVector::print(raw_ostream &OS) const {
- for (auto [Idx, Action] : enumerate(Actions)) {
- Action->print(OS);
- OS << "\n";
- }
-}
-void BottomUpVec::ActionsVector::dump() const { print(dbgs()); }
-#endif // NDEBUG
-
-void BottomUpVec::emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl,
- Value *Vec) {
- // Find where we should emit the unpacks.
- BasicBlock::iterator WhereIt;
- if (auto *VecI = dyn_cast<Instruction>(Vec)) {
- WhereIt = std::next(VecI->getIterator());
- } else {
- // If Vec is a constant then it should be safe to emit the unpacks at the
- // top of the block.
- // Note: Extracts from constants are usually folded to constants.
- assert(isa<Constant>(Vec) && "Expected constant!");
- assert(isa<Instruction>(Bndl[0]) &&
- "A widened Bndl should contain instrs!");
- BasicBlock *BB = cast<Instruction>(Bndl[0])->getParent();
- WhereIt =
- BB->empty()
- ? BB->begin()
- : std::next(
- VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
- }
-
- for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
- for (User *U : Elm->users()) {
- // Skip users that we just vectorized.
- if (IMaps->isVectorized(U))
- continue;
- auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
- Elm->replaceAllUsesWith(LastUnpackV);
- }
- }
-}
-
Value *BottomUpVec::emitVectors() {
Value *NewVec = nullptr;
for (const auto &ActionPtr : Actions) {
@@ -405,6 +125,7 @@ Value *BottomUpVec::emitVectors() {
break;
}
NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewVec << "\n");
// Collect any potentially dead scalar instructions, including the
// original scalars and pointer operands of loads/stores.
if (NewVec != nullptr)
@@ -529,7 +250,7 @@ bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl,
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "BottomUpVec: Vectorization Actions:\n";
Actions.dump());
emitVectors();
- tryEraseDeadInstrs();
+ tryEraseDeadInstrs(DEBUG_PREFIX);
return Change;
}
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index 10ba595910ee9..40d03c3886c0b 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -26,6 +26,7 @@ REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
REGION_PASS("tr-revert", ::llvm::sandboxir::TransactionAlwaysRevert)
REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
+REGION_PASS("top-down-vec", ::llvm::sandboxir::TopDownVec)
REGION_PASS("load-store-vec", ::llvm::sandboxir::LoadStoreVec)
#undef REGION_PASS
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.cpp
new file mode 100644
index 0000000000000..aafb7d9a54427
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.cpp
@@ -0,0 +1,262 @@
+//===- TopDownVec.cpp - A top-down vectorizer pass ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/SandboxIR/Function.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/Module.h"
+#include "llvm/SandboxIR/Region.h"
+#include "llvm/SandboxIR/Utils.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Debug.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+namespace llvm {
+
+#ifndef NDEBUG
+static cl::opt<bool>
+ AlwaysVerify("sbvec-topdown-always-verify", cl::init(false), cl::Hidden,
+ cl::desc("Helps find bugs by verifying the IR whenever we "
+ "emit new instructions (*very* expensive)."));
+#endif // NDEBUG
+
+static constexpr unsigned long StopAtDisabled =
+ std::numeric_limits<unsigned long>::max();
+static cl::opt<unsigned long>
+ StopAt("sbvec-topdown-stop-at", cl::init(StopAtDisabled), cl::Hidden,
+ cl::desc("Vectorize if the invocation count is < than this. 0 "
+ "disables vectorization."));
+
+static constexpr unsigned long StopBundleDisabled =
+ std::numeric_limits<unsigned long>::max();
+static cl::opt<unsigned long>
+ StopBundle("sbvec-topdown-stop-bndl", cl::init(StopBundleDisabled),
+ cl::Hidden, cl::desc("Vectorize up to this many bundles."));
+
+namespace sandboxir {
+
+Action *TopDownVec::vectorizeRec(ArrayRef<Value *> Bndl, unsigned Depth,
+ LegalityAnalysis &Legality) {
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
+ VecUtils::dump(Bndl));
+ const auto &LegalityRes =
+ Legality.canVectorize(Bndl, /*SkipScheduling=*/true);
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
+ auto ActionPtr =
+ std::make_unique<Action>(&LegalityRes, Bndl, ArrayRef<Value *>(), Depth);
+
+ if (LegalityRes.getSubclassID() == LegalityResultID::Widen) {
+ IMaps->registerVector(Bndl, ActionPtr.get());
+ }
+
+ // Pre-order push so defs are before uses.
+ Action *Action = ActionPtr.get();
+ Actions.push_back(std::move(ActionPtr));
+
+ if (LegalityRes.getSubclassID() == LegalityResultID::Widen) {
+ // Find users of Bndl to recurse down.
+ // Group the first user of each element if they match.
+ SmallVector<Value *, 4> UserBndl;
+ bool CanFormUserBndl = true;
+ for (Value *V : Bndl) {
+ if (V->user_begin() == V->user_end()) {
+ CanFormUserBndl = false;
+ break;
+ }
+ UserBndl.push_back(*V->user_begin());
+ }
+
+ if (CanFormUserBndl) {
+ auto *U0 = dyn_cast<Instruction>(UserBndl[0]);
+ if (!U0 || IMaps->isVectorized(U0))
+ CanFormUserBndl = false;
+ else {
+ for (Value *U : drop_begin(UserBndl)) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || UI->getOpcode() != U0->getOpcode() ||
+ UI->getType() != U0->getType() || IMaps->isVectorized(UI)) {
+ CanFormUserBndl = false;
+ break;
+ }
+ }
+ }
+ }
+
+ if (CanFormUserBndl) {
+ vectorizeRec(UserBndl, Depth + 1, Legality);
+ }
+ }
+
+ return Action;
+}
+
+Value *TopDownVec::emitVectors() {
+ Value *NewVec = nullptr;
+ for (const auto &ActionPtr : Actions) {
+ ArrayRef<Value *> Bndl = ActionPtr->Bndl;
+ const LegalityResult &LegalityRes = *ActionPtr->LegalityRes;
+ unsigned Depth = ActionPtr->Depth;
+ auto *UserBB = cast<Instruction>(Bndl[0])->getParent();
+
+ switch (LegalityRes.getSubclassID()) {
+ case LegalityResultID::Widen: {
+ auto *I = cast<Instruction>(Bndl[0]);
+ SmallVector<Value *, 2> VecOperands;
+ switch (I->getOpcode()) {
+ case Instruction::Opcode::Load:
+ VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+ break;
+ case Instruction::Opcode::Store: {
+ auto OpBndl = getOperand(Bndl, 0);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0])) {
+ VecOperands.push_back(OpA->Vec);
+ } else {
+ Value *Packed = createPack(OpBndl, UserBB);
+ VecOperands.push_back(Packed);
+ }
+ VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+ break;
+ }
+ default:
+ // Visit all operands and gather vectorized inputs.
+ for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) {
+ SmallVector<Value *, 4> OpBndl = getOperand(Bndl, OpIdx);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0])) {
+ VecOperands.push_back(OpA->Vec);
+ } else {
+ Value *Packed = createPack(OpBndl, UserBB);
+ VecOperands.push_back(Packed);
+ }
+ }
+ break;
+ }
+ NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewVec << "\n");
+ if (NewVec != nullptr)
+ collectPotentiallyDeadInstrs(Bndl);
+
+ emitUnpacksForExternalUses(ActionPtr->Bndl, NewVec);
+ break;
+ }
+ case LegalityResultID::DiamondReuse: {
+ NewVec = cast<DiamondReuse>(LegalityRes).getVector()->Vec;
+ break;
+ }
+ case LegalityResultID::DiamondReuseWithShuffle: {
+ auto *VecOp = cast<DiamondReuseWithShuffle>(LegalityRes).getVector()->Vec;
+ const ShuffleMask &Mask =
+ cast<DiamondReuseWithShuffle>(LegalityRes).getMask();
+ NewVec = createShuffle(VecOp, Mask, UserBB);
+ break;
+ }
+ case LegalityResultID::DiamondReuseMultiInput: {
+ const auto &Descr =
+ cast<DiamondReuseMultiInput>(LegalityRes).getCollectDescr();
+ Type *ResTy = VecUtils::getWideType(Bndl[0]->getType(), Bndl.size());
+
+ SmallVector<Value *, 4> DescrInstrs;
+ for (const auto &ElmDescr : Descr.getDescrs()) {
+ auto *V = ElmDescr.needsExtract() ? ElmDescr.getValue()->Vec
+ : ElmDescr.getScalar();
+ if (auto *Inst = dyn_cast<Instruction>(V))
+ DescrInstrs.push_back(Inst);
+ }
+ BasicBlock::iterator WhereIt =
+ getInsertPointAfterInstrs(DescrInstrs, UserBB);
+
+ Value *LastV = PoisonValue::get(ResTy);
+ Context &Ctx = LastV->getContext();
+ unsigned Lane = 0;
+ for (const auto &ElmDescr : Descr.getDescrs()) {
+ Value *VecOp = nullptr;
+ Value *ValueToInsert;
+ if (ElmDescr.needsExtract()) {
+ VecOp = ElmDescr.getValue()->Vec;
+ ConstantInt *IdxC =
+ ConstantInt::get(Type::getInt32Ty(Ctx), ElmDescr.getExtractIdx());
+ ValueToInsert = ExtractElementInst::create(
+ VecOp, IdxC, WhereIt, VecOp->getContext(), "VExt");
+ } else {
+ ValueToInsert = ElmDescr.getScalar();
+ }
+ auto NumLanesToInsert = VecUtils::getNumLanes(ValueToInsert);
+ if (NumLanesToInsert == 1) {
+ ConstantInt *LaneC = ConstantInt::get(Type::getInt32Ty(Ctx), Lane);
+ LastV = InsertElementInst::create(LastV, ValueToInsert, LaneC,
+ WhereIt, Ctx, "VIns");
+ } else {
+ for (unsigned LnCnt = 0; LnCnt != NumLanesToInsert; ++LnCnt) {
+ auto *ExtrIdxC = ConstantInt::get(Type::getInt32Ty(Ctx), LnCnt);
+ auto *ExtrI = ExtractElementInst::create(ValueToInsert, ExtrIdxC,
+ WhereIt, Ctx, "VExt");
+ unsigned InsLane = Lane + LnCnt;
+ auto *InsLaneC = ConstantInt::get(Type::getInt32Ty(Ctx), InsLane);
+ LastV = InsertElementInst::create(LastV, ExtrI, InsLaneC, WhereIt,
+ Ctx, "VIns");
+ }
+ }
+ Lane += NumLanesToInsert;
+ }
+ NewVec = LastV;
+ break;
+ }
+ case LegalityResultID::Pack: {
+ if (Depth == 0)
+ return nullptr;
+ NewVec = createPack(Bndl, UserBB);
+ break;
+ }
+ }
+ if (NewVec != nullptr) {
+ Change = true;
+ ActionPtr->Vec = NewVec;
+ }
+#ifndef NDEBUG
+ if (AlwaysVerify) {
+ Instruction *I0 = cast<Instruction>(Bndl[0]);
+ assert(!Utils::verifyFunction(I0->getParent()->getParent(), dbgs()) &&
+ "Broken function!");
+ }
+#endif // NDEBUG
+ }
+ return NewVec;
+}
+
+bool TopDownVec::tryVectorize(ArrayRef<Value *> Bndl,
+ LegalityAnalysis &Legality) {
+ Change = false;
+ if (LLVM_UNLIKELY(TopDownInvocationCnt++ >= StopAt &&
+ StopAt != StopAtDisabled))
+ return false;
+ DeadInstrCandidates.clear();
+ Legality.clear();
+ Actions.clear();
+ DebugBndlCnt = 0;
+ vectorizeRec(Bndl, /*Depth=*/0, Legality);
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "TopDownVec: Vectorization Actions:\n";
+ Actions.dump());
+ emitVectors();
+ tryEraseDeadInstrs(DEBUG_PREFIX);
+ return Change;
+}
+
+bool TopDownVec::runOnRegion(Region &Rgn, const Analyses &A) {
+ const auto &SeedSlice = Rgn.getAux();
+ assert(SeedSlice.size() >= 2 && "Bad slice!");
+ Function &F = *SeedSlice[0]->getParent()->getParent();
+ IMaps = std::make_unique<InstrMaps>();
+ LegalityAnalysis Legality(A.getAA(), A.getScalarEvolution(),
+ F.getParent()->getDataLayout(), F.getContext(),
+ *IMaps);
+
+ SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
+ return tryVectorize(SeedSliceVals, Legality);
+}
+
+} // namespace sandboxir
+} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.cpp
new file mode 100644
index 0000000000000..afea3508cf682
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.cpp
@@ -0,0 +1,257 @@
+//===- VecPassBase.cpp - Shared base for vectorizer passes ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/VecPassBase.h"
+#include "llvm/SandboxIR/Utils.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "sandbox-vectorizer"
+
+namespace llvm::sandboxir {
+
+#ifndef NDEBUG
+void VecPassBase::ActionsVector::print(raw_ostream &OS) const {
+ for (auto [Idx, Action] : enumerate(Actions)) {
+ Action->print(OS);
+ OS << "\n";
+ }
+}
+void VecPassBase::ActionsVector::dump() const { print(dbgs()); }
+#endif // NDEBUG
+
+Value *VecPassBase::createVectorInstr(ArrayRef<Value *> Bndl,
+ ArrayRef<Value *> Operands) {
+ assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
+ "Expect Instructions!");
+ auto &Ctx = Bndl[0]->getContext();
+
+ Type *ScalarTy = VecUtils::getElementType(Utils::getExpectedType(Bndl[0]));
+ auto *VecTy = VecUtils::getWideType(ScalarTy, VecUtils::getNumLanes(Bndl));
+
+ BasicBlock::iterator WhereIt =
+ getInsertPointAfterInstrs(Bndl, cast<Instruction>(Bndl[0])->getParent());
+
+ auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
+ Value *NewI = nullptr;
+ switch (Opcode) {
+ case Instruction::Opcode::ZExt:
+ case Instruction::Opcode::SExt:
+ case Instruction::Opcode::FPToUI:
+ case Instruction::Opcode::FPToSI:
+ case Instruction::Opcode::FPExt:
+ case Instruction::Opcode::PtrToInt:
+ case Instruction::Opcode::IntToPtr:
+ case Instruction::Opcode::SIToFP:
+ case Instruction::Opcode::UIToFP:
+ case Instruction::Opcode::Trunc:
+ case Instruction::Opcode::FPTrunc:
+ case Instruction::Opcode::BitCast: {
+ assert(Operands.size() == 1u && "Casts are unary!");
+ NewI = CastInst::create(VecTy, Opcode, Operands[0], WhereIt, Ctx, "VCast");
+ break;
+ }
+ case Instruction::Opcode::FCmp:
+ case Instruction::Opcode::ICmp: {
+ auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
+ assert(all_of(drop_begin(Bndl),
+ [Pred](auto *SBV) {
+ return cast<CmpInst>(SBV)->getPredicate() == Pred;
+ }) &&
+ "Expected same predicate across bundle.");
+ NewI =
+ CmpInst::create(Pred, Operands[0], Operands[1], WhereIt, Ctx, "VCmp");
+ break;
+ }
+ case Instruction::Opcode::Select: {
+ NewI = SelectInst::create(Operands[0], Operands[1], Operands[2], WhereIt,
+ Ctx, "Vec");
+ break;
+ }
+ case Instruction::Opcode::FNeg: {
+ auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
+ auto OpC = UOp0->getOpcode();
+ NewI = UnaryOperator::createWithCopiedFlags(OpC, Operands[0], UOp0, WhereIt,
+ Ctx, "Vec");
+ break;
+ }
+ case Instruction::Opcode::Add:
+ case Instruction::Opcode::FAdd:
+ case Instruction::Opcode::Sub:
+ case Instruction::Opcode::FSub:
+ case Instruction::Opcode::Mul:
+ case Instruction::Opcode::FMul:
+ case Instruction::Opcode::UDiv:
+ case Instruction::Opcode::SDiv:
+ case Instruction::Opcode::FDiv:
+ case Instruction::Opcode::URem:
+ case Instruction::Opcode::SRem:
+ case Instruction::Opcode::FRem:
+ case Instruction::Opcode::Shl:
+ case Instruction::Opcode::LShr:
+ case Instruction::Opcode::AShr:
+ case Instruction::Opcode::And:
+ case Instruction::Opcode::Or:
+ case Instruction::Opcode::Xor: {
+ auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
+ auto *LHS = Operands[0];
+ auto *RHS = Operands[1];
+ NewI = BinaryOperator::createWithCopiedFlags(BinOp0->getOpcode(), LHS, RHS,
+ BinOp0, WhereIt, Ctx, "Vec");
+ break;
+ }
+ case Instruction::Opcode::Load: {
+ auto *Ld0 = cast<LoadInst>(Bndl[0]);
+ Value *Ptr = Ld0->getPointerOperand();
+ NewI = LoadInst::create(VecTy, Ptr, Ld0->getAlign(), WhereIt, Ctx, "VecL");
+ break;
+ }
+ case Instruction::Opcode::Store: {
+ auto Align = cast<StoreInst>(Bndl[0])->getAlign();
+ Value *Val = Operands[0];
+ Value *Ptr = Operands[1];
+ NewI = StoreInst::create(Val, Ptr, Align, WhereIt, Ctx);
+ break;
+ }
+ case Instruction::Opcode::UncondBr:
+ case Instruction::Opcode::CondBr:
+ case Instruction::Opcode::Ret:
+ case Instruction::Opcode::PHI:
+ case Instruction::Opcode::AddrSpaceCast:
+ case Instruction::Opcode::Call:
+ case Instruction::Opcode::GetElementPtr:
+ llvm_unreachable("Unimplemented");
+ break;
+ default:
+ llvm_unreachable("Unimplemented");
+ break;
+ }
+ return NewI;
+}
+
+Value *VecPassBase::createShuffle(Value *VecOp, const ShuffleMask &Mask,
+ BasicBlock *UserBB) {
+ BasicBlock::iterator WhereIt = getInsertPointAfterInstrs({VecOp}, UserBB);
+ return ShuffleVectorInst::create(VecOp, VecOp, Mask, WhereIt,
+ VecOp->getContext(), "VShuf");
+}
+
+Value *VecPassBase::createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB) {
+ BasicBlock::iterator WhereIt = getInsertPointAfterInstrs(ToPack, UserBB);
+
+ Type *ScalarTy = VecUtils::getCommonScalarType(ToPack);
+ unsigned Lanes = VecUtils::getNumLanes(ToPack);
+ Type *VecTy = VecUtils::getWideType(ScalarTy, Lanes);
+
+ Value *LastInsert = PoisonValue::get(VecTy);
+ Context &Ctx = ToPack[0]->getContext();
+ unsigned InsertIdx = 0;
+
+ for (Value *Elm : ToPack) {
+ if (Elm->getType()->isVectorTy()) {
+ unsigned NumElms =
+ cast<FixedVectorType>(Elm->getType())->getNumElements();
+ for (auto ExtrLane : seq<int>(0, NumElms)) {
+ Constant *ExtrLaneC =
+ ConstantInt::getSigned(Type::getInt32Ty(Ctx), ExtrLane);
+ auto *ExtrI =
+ ExtractElementInst::create(Elm, ExtrLaneC, WhereIt, Ctx, "VPack");
+ if (!isa<Constant>(ExtrI))
+ WhereIt = std::next(cast<Instruction>(ExtrI)->getIterator());
+ Constant *InsertLaneC =
+ ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
+ LastInsert = InsertElementInst::create(LastInsert, ExtrI, InsertLaneC,
+ WhereIt, Ctx, "VPack");
+ if (!isa<Constant>(LastInsert))
+ WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
+ }
+ } else {
+ Constant *InsertLaneC =
+ ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
+ LastInsert = InsertElementInst::create(LastInsert, Elm, InsertLaneC,
+ WhereIt, Ctx, "VPack");
+ if (!isa<Constant>(LastInsert))
+ WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
+ }
+ }
+ return LastInsert;
+}
+
+void VecPassBase::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
+ for (Value *V : Bndl)
+ DeadInstrCandidates.insert(cast<Instruction>(V));
+ auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
+ switch (Opcode) {
+ case Instruction::Opcode::Load: {
+ for (Value *V : drop_begin(Bndl))
+ if (auto *Ptr =
+ dyn_cast<Instruction>(cast<LoadInst>(V)->getPointerOperand()))
+ DeadInstrCandidates.insert(Ptr);
+ break;
+ }
+ case Instruction::Opcode::Store: {
+ for (Value *V : drop_begin(Bndl))
+ if (auto *Ptr =
+ dyn_cast<Instruction>(cast<StoreInst>(V)->getPointerOperand()))
+ DeadInstrCandidates.insert(Ptr);
+ break;
+ }
+ default:
+ break;
+ }
+}
+
+void VecPassBase::tryEraseDeadInstrs(StringRef PassName) {
+ DenseMap<BasicBlock *, SmallVector<Instruction *>> SortedDeadInstrCandidates;
+ for (auto *DeadI : DeadInstrCandidates)
+ SortedDeadInstrCandidates[DeadI->getParent()].push_back(DeadI);
+ for (auto &Pair : SortedDeadInstrCandidates)
+ sort(Pair.second,
+ [](Instruction *I1, Instruction *I2) { return I1->comesBefore(I2); });
+ for (const auto &Pair : SortedDeadInstrCandidates) {
+ for (Instruction *I : reverse(Pair.second)) {
+ if (I->hasNUses(0)) {
+ LLVM_DEBUG(dbgs() << PassName << "Erase dead: " << *I << "\n");
+ I->eraseFromParent();
+ }
+ }
+ }
+ DeadInstrCandidates.clear();
+}
+
+void VecPassBase::emitUnpacksForExternalUses(ArrayRef<Value *> Bndl,
+ Value *Vec) {
+ BasicBlock::iterator WhereIt;
+ if (auto *VecI = dyn_cast<Instruction>(Vec)) {
+ WhereIt = std::next(VecI->getIterator());
+ } else {
+ // If Vec is a constant then it should be safe to emit the unpacks at the
+ // top of the block.
+ // Note: Extracts from constants are usually folded to constants.
+ assert(isa<Constant>(Vec) && "Expected constant!");
+ assert(isa<Instruction>(Bndl[0]) &&
+ "A widened Bndl should contain instrs!");
+ BasicBlock *BB = cast<Instruction>(Bndl[0])->getParent();
+ WhereIt =
+ BB->empty()
+ ? BB->begin()
+ : std::next(
+ VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
+ }
+
+ for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
+ for (User *U : Elm->users()) {
+ // Skip users that we just vectorized.
+ if (IMaps->isVectorized(U))
+ continue;
+ auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
+ Elm->replaceAllUsesWith(LastUnpackV);
+ }
+ }
+}
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 9326869461082..e53895660859c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -9,6 +9,7 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TopDownVec.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysRevert.h"
diff --git a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
index 593965ab01680..624cf3d918362 100644
--- a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
; Checks the handling of users outside the vectorized graph.
@@ -13,6 +14,20 @@ define void @external_users(ptr %ptr) {
; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
; CHECK-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub float [[LD1]], 0.000000e+00
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x float> [[VPACK]], float [[SUB1]], i32 1, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: store <2 x float> [[VPACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -35,6 +50,17 @@ define void @external_user_of_constant(ptr %ptr, ptr %ptrX) {
; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
; CHECK-NEXT: store i32 0, ptr [[PTRX]], align 4
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_user_of_constant(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], ptr [[PTRX:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[ZEXT0:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: [[ZEXT1:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i32> poison, i32 [[ZEXT0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i32> [[VPACK]], i32 [[ZEXT1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: store <2 x i32> [[VPACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: store i32 [[ZEXT0]], ptr [[PTRX]], align 4
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -57,6 +83,23 @@ define void @vector_external_users(ptr %ptr) {
; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACKINS2]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <3 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK2:%.*]] = insertelement <3 x float> [[VPACK]], float [[VPACK1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK3:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK4:%.*]] = insertelement <3 x float> [[VPACK2]], float [[VPACK3]], i32 2, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: store <3 x float> [[VPACK4]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -80,6 +123,26 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK-NEXT: store <4 x float> [[VEC]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACK]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users_lane_and_index_differ(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load <2 x float>, ptr [[PTR0]], align 8
+; TOPDOWN-NEXT: [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub <2 x float> [[LD0]], zeroinitializer
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT: [[VPACK:%.*]] = extractelement <2 x float> [[SUB0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <4 x float> poison, float [[VPACK]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK2:%.*]] = extractelement <2 x float> [[SUB0]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK3:%.*]] = insertelement <4 x float> [[VPACK1]], float [[VPACK2]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK4:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK5:%.*]] = insertelement <4 x float> [[VPACK3]], float [[VPACK4]], i32 2, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK6:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK7:%.*]] = insertelement <4 x float> [[VPACK5]], float [[VPACK6]], i32 3, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: store <4 x float> [[VPACK7]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr <2 x float>, ptr %ptr, i32 0
%ptr1 = getelementptr <2 x float>, ptr %ptr, i32 1
@@ -99,3 +162,8 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index 743d705fd48ff..2a50ba84e53db 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
@@ -7,6 +8,12 @@ define void @pack_constants(ptr %ptr) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @pack_constants(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
%ptr1 = getelementptr i8, ptr %ptr, i32 1
@@ -27,14 +34,31 @@ define void @packPHIs(ptr %ptr) {
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packPHIs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
br label %loop
@@ -63,14 +87,31 @@ define void @packFromOtherBB(ptr %ptr, i8 %val) {
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromOtherBB(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[VAL:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[VAL]], 0
+; TOPDOWN-NEXT: [[MUL1:%.*]] = mul i8 [[VAL]], 1
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %val, 0
%mul1 = mul i8 %val, 1
@@ -97,12 +138,25 @@ define void @packFromDiffBBs(ptr %ptr, i8 %v) {
; CHECK-NEXT: br label %[[BB:.*]]
; CHECK: [[BB]]:
; CHECK-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromDiffBBs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[V:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*:]]
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[V]], 1
+; TOPDOWN-NEXT: br label %[[BB:.*]]
+; TOPDOWN: [[BB]]:
+; TOPDOWN-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %v, 1
br label %bb
@@ -121,3 +175,8 @@ bb:
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
new file mode 100644
index 0000000000000..a9db7d9225685
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 \
+; RUN: -sbvec-collect-seeds=loads \
+; RUN: -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" \
+; RUN: %s -S | FileCheck %s
+
+define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_fadd_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META0]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META0]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define void @load_chain_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_chain_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fmul <2 x float> [[VECL]], splat (float 3.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[VEC1:%.*]] = fadd <2 x float> [[VEC]], splat (float 2.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC1]], ptr [[PTR2_0]], align 4, !sandboxvec [[META1]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fmul0 = fmul float %ld0, 3.0
+ %fmul1 = fmul float %ld1, 3.0
+
+ %fadd0 = fadd float %fmul0, 2.0
+ %fadd1 = fadd float %fmul1, 2.0
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define float @load_fadd_external_use(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define float @load_fadd_external_use(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VEC]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: store float [[UNPACK]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[UNPACK1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret float [[UNPACK]]
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+
+ ret float %fadd0
+}
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+; CHECK: [[META2]] = distinct !{!"sandboxregion"}
+;.
More information about the llvm-commits
mailing list