[llvm-branch-commits] [llvm] [SandboxVectorizer] Implement topdown/bottomup vectorizers in unison (PR #205249)
Anshil Gandhi via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Jul 29 22:50:31 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/205249
>From e6c5169218507c5cea31a1cdc54711a99af47350 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Wed, 29 Jul 2026 16:45:40 -0400
Subject: [PATCH 01/11] [SandboxIR] Fix notifyEraseInstr to skip scheduled
neighbors
Guard both loops with !PredN->scheduled() / !SuccN->scheduled() so
scheduled neighbors are left untouched, and add a unit test that erases
a node with one scheduled and one unscheduled predecessor to cover the
fix.
---
.../SandboxVectorizer/DependencyGraph.cpp | 6 ++-
.../SandboxVectorizer/DependencyGraphTest.cpp | 43 +++++++++++++++++++
2 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
index 11149a16b044f..123b4f038371b 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
@@ -583,9 +583,11 @@ void DependencyGraph::notifyEraseInstr(Instruction *I) {
// If this is a non-mem node we only need to update UnscheduledSuccs.
if (!N->scheduled()) {
for (auto *PredN : N->preds(*this))
- PredN->decrUnscheduledSuccs();
+ if (!PredN->scheduled())
+ PredN->decrUnscheduledSuccs();
for (auto *SuccN : N->succs(*this))
- SuccN->decrUnscheduledPreds();
+ if (!SuccN->scheduled())
+ SuccN->decrUnscheduledPreds();
}
}
// Finally erase the Node.
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
index 831976875f8d3..20bd959c1b107 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -1516,3 +1516,46 @@ define void @foo(i8 %v0) {
Add0->setOperand(0, Sched);
EXPECT_EQ(Add0N->getNumUnscheduledPreds(), 0u);
}
+
+// When erasing a non-mem instruction we must not touch the
+// UnscheduledSuccs of an already-scheduled predecessor, since that counter
+// is set to std::nullopt once a node is scheduled.
+TEST_F(DependencyGraphTest, EraseInstrCallbackNonMemWithScheduledPred) {
+ parseIR(C, R"IR(
+define void @foo(i8 %v0) {
+ %predSched = add i8 %v0, 0
+ %predUnsched = add i8 %v0, 1
+ %n = add i8 %predSched, %predUnsched
+ ret void
+}
+)IR");
+ llvm::Function *LLVMF = &*M->getFunction("foo");
+ sandboxir::Context Ctx(C);
+ auto *F = Ctx.createFunction(LLVMF);
+ auto *BB = &*F->begin();
+ auto It = BB->begin();
+ auto *PredSched = cast<sandboxir::BinaryOperator>(&*It++);
+ auto *PredUnsched = cast<sandboxir::BinaryOperator>(&*It++);
+ auto *N = cast<sandboxir::BinaryOperator>(&*It++);
+
+ sandboxir::DependencyGraph DAG(getAA(*LLVMF), Ctx);
+ DAG.extend({PredSched, N});
+ auto *PredSchedN = DAG.getNode(PredSched);
+ auto *PredUnschedN = DAG.getNode(PredUnsched);
+ EXPECT_EQ(PredSchedN->getNumUnscheduledSuccs(), 1u);
+ EXPECT_EQ(PredUnschedN->getNumUnscheduledSuccs(), 1u);
+
+ // Mark one of N's predecessors as scheduled. Its UnscheduledSuccs becomes
+ // std::nullopt.
+ PredSchedN->setScheduled();
+
+ // Erase N, which is *not* scheduled. This must not attempt to decrement
+ // the (now invalid) UnscheduledSuccs of PredSchedN, but should still
+ // update the counter of the unscheduled predecessor.
+ N->eraseFromParent();
+ EXPECT_EQ(DAG.getNode(N), nullptr);
+ EXPECT_EQ(PredUnschedN->getNumUnscheduledSuccs(), 0u);
+#ifndef NDEBUG
+ EXPECT_FALSE(PredSchedN->validUnscheduledSuccs());
+#endif
+}
>From 345444aa20dbe1a582b0a737ab04b9187d400246 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Sun, 14 Jun 2026 17:01:48 -0400
Subject: [PATCH 02/11] [SBVec] Add top-down vectorization to the unified
Sandbox Vectorizer
Extend the Sandbox Vectorizer's `bottom-up-vec` pass so a single
implementation can vectorize in either direction, and add the top-down
strategy that walks def-use chains forward from a seed.
Direction selection
--------------------
The pass direction is chosen from the Region's auxiliary pass argument:
"bottom-up" (or empty, the default) and "top-down" map onto a
SchedDirection, and any other value is rejected with a fatal usage error.
The vectorizer always runs in the same direction as the scheduler.
Top-down traversal
------------------
Bottom-up starts from a seed slice (e.g. stores to consecutive addresses)
and recurses into operands. Top-down instead starts from a seed of
consecutive loads and recurses into *users*:
- vectorizeRec() registers the current bundle's vector (pre-order) before
recursing, so instructions are marked vectorized as soon as they are
claimed. This prevents sibling user bundles from claiming the same
instruction and guarantees termination.
- VecUtils::getNextUserBundles() drives the walk. For each user of lane 0
it tries to assemble a matching user for every remaining lane, requiring
the same opcode, type, parent block, and operand-usage indices, and
claiming each instruction at most once. Only complete bundles (one user
per lane) are returned.
- A non-Widen legality result stops the walk down that path: the bundle is
left scalar and no action is recorded. DiamondReuse results cannot occur
top-down because already-vectorized users are skipped, so a bundle never
contains an instruction already in InstrMaps.
Operand and external-use handling
---------------------------------
Because a user bundle is emitted after its operand bundle, emitVectors()
looks up each operand's vector in InstrMaps and creates a pack when the
operand was not vectorized. emitUnpacksForExternalUses() now redirects
only the genuinely external (non-vectorized) uses via replaceUsesWithIf(),
instead of a blanket replaceAllUsesWith() that would corrupt the operands
of user bundles not yet emitted. Scheduling is currently skipped for the
top-down direction (TODO).
Refactoring
-----------
Unify the two strategies to avoid code duplication: introduce a shared
BundleTy alias, move user-bundle collection into VecUtils (with unit
tests), and thread the direction through legality checks and vector
emission.
Co-authored-by: Cursor <cursoragent at cursor.com>
---
.../SandboxVectorizer/Passes/BottomUpVec.h | 45 +-
.../Vectorize/SandboxVectorizer/VecUtils.h | 9 +
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 117 +++--
.../Vectorize/SandboxVectorizer/VecUtils.cpp | 79 ++++
.../SandboxVectorizer/external_uses.ll | 68 +++
.../test/Transforms/SandboxVectorizer/pack.ll | 77 ++-
.../SandboxVectorizer/topdown_vec.ll | 441 ++++++++++++++++++
.../SandboxVectorizer/VecUtilsTest.cpp | 229 +++++++++
8 files changed, 1017 insertions(+), 48 deletions(-)
create mode 100644 llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index 50ac1ecbf9f23..bdddc38906023 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -6,7 +6,8 @@
//
//===----------------------------------------------------------------------===//
//
-// A Bottom-Up Vectorizer pass.
+// A vectorizer pass that walks the def-use chain bottom-up or top-down,
+// depending on the auxiliary pass argument.
//
#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
@@ -16,24 +17,33 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/SandboxIR/Constant.h"
#include "llvm/SandboxIR/Pass.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h"
namespace llvm::sandboxir {
-/// This is a simple bottom-up vectorizer Region pass.
+/// This is a simple bottom-up (top-down) vectorizer Region pass.
/// It expects a "seed slice" as an input in the Region's Aux vector.
/// The "seed slice" is a vector of instructions that can be used as a starting
-/// point for vectorization, like stores to consecutive memory addresses.
-/// Starting from the seed instructions, it walks up the def-use chain looking
-/// for more instructions that can be vectorized. This pass will generate vector
-/// code if it can legally vectorize the code, regardless of whether it is
-/// profitable or not. For now profitability is checked at the end of the region
-/// pass pipeline by a dedicated pass that accepts or rejects the IR
-/// transaction, depending on the cost.
+/// point for vectorization, like stores (loads) to consecutive memory
+/// addresses. Starting from the seed instructions, it walks up (down) the
+/// def-use (use-def) chains looking for more instructions that can be
+/// vectorized. This pass will generate vector code if it can legally vectorize
+/// the code, regardless of whether it is profitable or not. For now
+/// profitability is checked at the end of the region pass pipeline by a
+/// dedicated pass that accepts or rejects the IR transaction, depending on the
+/// cost.
class LLVM_ABI BottomUpVec final : public RegionPass {
+private:
+ /// Set to true whenever the pass modifies the IR.
bool Change = false;
+ static constexpr StringRef TopDownArgStr = "top-down";
+ static constexpr StringRef BottomUpArgStr = "bottom-up";
+ /// Direction for vectorization, defaults to bottom-up.
+ SchedDirection Dir = SchedDirection::BottomUp;
/// The original instructions that are potentially dead after vectorization.
DenseSet<Instruction *> DeadInstrCandidates;
/// Maps scalars to vectors.
@@ -84,8 +94,8 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
/// vectorize in vectorizeRec().
unsigned DebugBndlCnt = 0;
- /// Recursively try to vectorize \p Bndl and its operands. This populates the
- /// `Actions` vector.
+ /// Recursively try to vectorize \p Bndl. For bottom-up vectorization \p
+ /// UserBndl tracks the bundle of users that led to this recursion.
Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
unsigned Depth, LegalityAnalysis &Legality);
/// If the values in \p Bndl have external users, then emit unpacks and
@@ -99,8 +109,19 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
public:
BottomUpVec(StringRef AuxArg) : RegionPass("bottom-up-vec") {
- assert(AuxArg.empty() && "This pass ignores aux arg!");
+ if (AuxArg.empty() || AuxArg == BottomUpArgStr) {
+ Dir = SchedDirection::BottomUp;
+ } else if (AuxArg == TopDownArgStr) {
+ Dir = SchedDirection::TopDown;
+ } else {
+ std::string ErrStr;
+ raw_string_ostream ErrSS(ErrStr);
+ ErrSS << "bottom-up-vec only supports '" << BottomUpArgStr << "' or '"
+ << TopDownArgStr << "' aux argument!\n";
+ reportFatalUsageError(ErrStr.c_str());
+ }
}
+
bool runOnRegion(Region &Rgn, const Analyses &A) final;
};
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 803718fc33792..8f37d2bafbd79 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -33,6 +33,10 @@ template <> struct DenseMapInfo<SmallVector<sandboxir::Value *>> {
namespace sandboxir {
+class InstrMaps;
+
+using BundleTy = SmallVector<Value *, 4>;
+
class VecUtils {
public:
/// \Returns the number of elements in \p Ty. That is the number of lanes if a
@@ -225,6 +229,11 @@ class VecUtils {
/// \Returns the first integer power of 2 that is <= Num.
LLVM_ABI static unsigned getFloorPowerOf2(unsigned Num);
+ /// For each user of lane 0 in \p Bndl, try to form a bundle of matching
+ /// users for all lanes. Returns all complete user bundles found.
+ LLVM_ABI static SmallVector<BundleTy>
+ getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps);
+
/// Helper struct for `matchPack()`. Describes the instructions and operands
/// of a pack pattern.
struct PackPattern {
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index d0f8b7ad12e34..e7599035cc3a6 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -13,7 +13,9 @@
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
#include "llvm/SandboxIR/Utils.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Debug.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
namespace llvm {
@@ -40,9 +42,8 @@ static cl::opt<unsigned long>
namespace sandboxir {
-static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
- unsigned OpIdx) {
- SmallVector<Value *, 4> Operands;
+static BundleTy getOperand(ArrayRef<Value *> Bndl, unsigned OpIdx) {
+ BundleTy Operands;
for (Value *BndlV : Bndl) {
auto *BndlI = cast<Instruction>(BndlV);
Operands.push_back(BndlI->getOperand(OpIdx));
@@ -285,9 +286,42 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
DebugBndlCnt++ >= StopBundle && StopBundle != StopBundleDisabled;
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
VecUtils::dump(Bndl));
+ /// TODO: Enable scheduling for topdown vectorization
const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
: Legality.canVectorize(Bndl);
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
+
+ if (Dir == SchedDirection::TopDown) {
+ // A non-Widen result means we can't extend the vectorized region into
+ // this bundle, so leave its instructions scalar and don't record an
+ // action for it. The scalar users of the already-widened defs get their
+ // values through the unpacks emitted by emitUnpacksForExternalUses().
+ // Note: The DiamondReuse* results are unreachable in the top-down
+ // direction because getNextUserBundles() skips already-vectorized users,
+ // so a bundle never contains instructions registered in IMaps.
+ if (LegalityRes.getSubclassID() != LegalityResultID::Widen)
+ return nullptr;
+
+ auto ActionPtr = std::make_unique<Action>(&LegalityRes, Bndl,
+ ArrayRef<Value *>(), Depth);
+ Action *Action = ActionPtr.get();
+ IMaps->registerVector(Bndl, Action);
+ Actions.push_back(std::move(ActionPtr));
+
+ // Walk down the def-use chain. Each lane in \p Bndl may feed several
+ // users, so we form every compatible user bundle and recurse into each
+ // one.
+ //
+ // Recursing right after forming each bundle marks its instructions as
+ // vectorized (pre-order registration), which prevents sibling bundles
+ // from claiming the same instruction and guarantees termination.
+ for (const auto &NextUserBndl : VecUtils::getNextUserBundles(Bndl, *IMaps))
+ vectorizeRec(NextUserBndl, Bndl, Depth + 1, Legality);
+
+ return Action;
+ }
+
+ // Bottom up direction
auto ActionPtr =
std::make_unique<Action>(&LegalityRes, Bndl, UserBndl, Depth);
SmallVector<Action *> Operands;
@@ -362,13 +396,19 @@ void BottomUpVec::emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl,
}
for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
- for (User *U : Elm->users()) {
- // Skip users that we just vectorized.
- if (IMaps->isVectorized(U))
- continue;
- auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
- Elm->replaceAllUsesWith(LastUnpackV);
- }
+ // Only redirect the external (non-vectorized) uses to an unpack and leave
+ // the vectorized users untouched. A blanket replaceAllUsesWith() would
+ // also rewrite the operands of users we are going to vectorize but have
+ // not emitted yet (in the top-down direction a user bundle is emitted
+ // after its operand bundle), which would corrupt those operands.
+ auto IsExternal = [this](const Use &U) {
+ return !IMaps->isVectorized(U.getUser());
+ };
+ // Don't emit a dead unpack if all uses are internal to the vector region.
+ if (none_of(Elm->uses(), IsExternal))
+ continue;
+ auto *UnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
+ Elm->replaceUsesWithIf(UnpackV, IsExternal);
}
}
@@ -387,22 +427,44 @@ Value *BottomUpVec::emitVectors() {
case LegalityResultID::Widen: {
auto *I = cast<Instruction>(Bndl[0]);
SmallVector<Value *, 2> VecOperands;
- switch (I->getOpcode()) {
- case Instruction::Opcode::Load:
- VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
- break;
- case Instruction::Opcode::Store: {
- VecOperands.push_back(ActionPtr->Operands[0]->Vec);
- VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
- break;
- }
- default:
- // Visit all operands.
- for (Action *OpA : ActionPtr->Operands) {
- auto *VecOp = OpA->Vec;
- VecOperands.push_back(VecOp);
+ if (Dir == SchedDirection::BottomUp) {
+ switch (I->getOpcode()) {
+ case Instruction::Opcode::Load:
+ VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+ break;
+ case Instruction::Opcode::Store:
+ VecOperands.push_back(ActionPtr->Operands[0]->Vec);
+ VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+ break;
+ default:
+ for (Action *OpA : ActionPtr->Operands)
+ VecOperands.push_back(OpA->Vec);
+ break;
+ }
+ } else {
+ switch (I->getOpcode()) {
+ case Instruction::Opcode::Load:
+ VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+ break;
+ case Instruction::Opcode::Store: {
+ auto OpBndl = getOperand(Bndl, 0);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+ VecOperands.push_back(OpA->Vec);
+ else
+ VecOperands.push_back(createPack(OpBndl, UserBB));
+ VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+ break;
+ }
+ default:
+ for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) {
+ BundleTy OpBndl = getOperand(Bndl, OpIdx);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+ VecOperands.push_back(OpA->Vec);
+ else
+ VecOperands.push_back(createPack(OpBndl, UserBB));
+ }
+ break;
}
- break;
}
NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
// Collect any potentially dead scalar instructions, including the
@@ -526,7 +588,8 @@ bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl,
Actions.clear();
DebugBndlCnt = 0;
vectorizeRec(Bndl, {}, /*Depth=*/0, Legality);
- LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "BottomUpVec: Vectorization Actions:\n";
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << schedDirectionToStr(Dir)
+ << "Vec: Vectorization Actions:\n";
Actions.dump());
emitVectors();
tryEraseDeadInstrs();
@@ -540,7 +603,7 @@ bool BottomUpVec::runOnRegion(Region &Rgn, const Analyses &A) {
IMaps = std::make_unique<InstrMaps>();
LegalityAnalysis Legality(A.getAA(), A.getScalarEvolution(),
F.getParent()->getDataLayout(), F.getContext(),
- *IMaps, SchedDirection::BottomUp);
+ *IMaps, Dir);
// TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
index 6f9ef07e467d2..9f5b5422b9b13 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
@@ -8,8 +8,87 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
+
namespace llvm::sandboxir {
+SmallVector<BundleTy> VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl,
+ const InstrMaps &IMaps) {
+ SmallVector<BundleTy> Bundles;
+ if (Bndl.empty())
+ return Bundles;
+
+ // Collect the operand indices at which \p U uses \p V. Operands are scanned
+ // in ascending order, so the result is sorted.
+ auto GetOpIdxVec = [](Value *V, User *U) -> SmallVector<unsigned, 2> {
+ SmallVector<unsigned, 2> OpIdxVec;
+ for (unsigned Idx : seq<unsigned>(U->getNumOperands()))
+ if (U->getOperand(Idx) == V)
+ OpIdxVec.push_back(Idx);
+ return OpIdxVec;
+ };
+
+ Value *V0 = Bndl[0];
+ DenseSet<User *> SeenUsers;
+ // For each user U0 of lane 0, try to form a bundle of matching users across
+ // all lanes.
+ for (User *U0 : V0->users()) {
+ if (!SeenUsers.insert(U0).second)
+ continue;
+ auto *UI0 = dyn_cast<Instruction>(U0);
+ if (!UI0 || IMaps.isVectorized(UI0))
+ continue;
+
+ // The operand indices at which lane 0's user U0 uses lane 0's value V0.
+ // Every other lane's user must use its lane value at the exact same operand
+ // indices; otherwise the widened user's operands can't be grouped
+ // consistently (each vector operand lane must come from the same position).
+ SmallVector<unsigned, 2> OpIdxVec0 = GetOpIdxVec(V0, UI0);
+ assert(!OpIdxVec0.empty() && "U0 does not use V0!");
+
+ // Find a distinct matching user for each of the remaining lanes.
+ BundleTy NextUserBndl;
+ NextUserBndl.push_back(UI0);
+ SmallPtrSet<Instruction *, 4> Claimed;
+ Claimed.insert(UI0);
+ for (Value *V : drop_begin(Bndl)) {
+ Instruction *Match = nullptr;
+ for (User *U : V->users()) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || IMaps.isVectorized(UI) || Claimed.contains(UI))
+ continue;
+ if (UI->getOpcode() != UI0->getOpcode() ||
+ UI->getType() != UI0->getType())
+ continue;
+ if (UI->getParent() != UI0->getParent())
+ continue;
+
+ // Require the same operand-usage pattern as lane 0 (same indices, in
+ // order). This rejects both operand-index mismatches and cases where V
+ // is used a different number of times than V0 is in U0.
+ if (GetOpIdxVec(V, UI) != OpIdxVec0)
+ continue;
+
+ Match = UI;
+ break;
+ }
+ if (!Match) {
+ NextUserBndl.clear();
+ break;
+ }
+ Claimed.insert(Match);
+ NextUserBndl.push_back(Match);
+ }
+
+ if (NextUserBndl.size() == Bndl.size())
+ Bundles.emplace_back(std::move(NextUserBndl));
+ }
+ return Bundles;
+}
+
unsigned VecUtils::getFloorPowerOf2(unsigned Num) {
if (Num == 0)
return Num;
diff --git a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
index 593965ab01680..a87498bf9549c 100644
--- a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec(top-down),tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
; Checks the handling of users outside the vectorized graph.
@@ -13,6 +14,20 @@ define void @external_users(ptr %ptr) {
; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
; CHECK-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub float [[LD1]], 0.000000e+00
+; TOPDOWN-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[SUB1]], i32 1, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: store <2 x float> [[PACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -35,6 +50,17 @@ define void @external_user_of_constant(ptr %ptr, ptr %ptrX) {
; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
; CHECK-NEXT: store i32 0, ptr [[PTRX]], align 4
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_user_of_constant(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], ptr [[PTRX:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[ZEXT0:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: [[ZEXT1:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: [[PACK:%.*]] = insertelement <2 x i32> poison, i32 [[ZEXT0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[PACK1:%.*]] = insertelement <2 x i32> [[PACK]], i32 [[ZEXT1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: store <2 x i32> [[PACK1]], ptr [[PTR0]], align 4, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: store i32 [[ZEXT0]], ptr [[PTRX]], align 4
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -57,6 +83,23 @@ define void @vector_external_users(ptr %ptr) {
; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACKINS2]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[SUB0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <3 x float> [[PACK]], float [[VPACK]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK2:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[VPACK3:%.*]] = insertelement <3 x float> [[VPACK1]], float [[VPACK2]], i32 2, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: store <3 x float> [[VPACK3]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -80,6 +123,26 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK-NEXT: store <4 x float> [[VEC]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACK]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users_lane_and_index_differ(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[LD0:%.*]] = load <2 x float>, ptr [[PTR0]], align 8
+; TOPDOWN-NEXT: [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub <2 x float> [[LD0]], zeroinitializer
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT: [[VPACK:%.*]] = extractelement <2 x float> [[SUB0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <4 x float> poison, float [[VPACK]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK2:%.*]] = extractelement <2 x float> [[SUB0]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK3:%.*]] = insertelement <4 x float> [[VPACK1]], float [[VPACK2]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK4:%.*]] = extractelement <2 x float> [[SUB1]], i32 0, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK5:%.*]] = insertelement <4 x float> [[VPACK3]], float [[VPACK4]], i32 2, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK6:%.*]] = extractelement <2 x float> [[SUB1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[VPACK7:%.*]] = insertelement <4 x float> [[VPACK5]], float [[VPACK6]], i32 3, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: store <4 x float> [[VPACK7]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr <2 x float>, ptr %ptr, i32 0
%ptr1 = getelementptr <2 x float>, ptr %ptr, i32 1
@@ -99,3 +162,8 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index 743d705fd48ff..9fde789e55196 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec(top-down),tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
@@ -7,6 +8,12 @@ define void @pack_constants(ptr %ptr) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @pack_constants(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
%ptr1 = getelementptr i8, ptr %ptr, i32 1
@@ -27,14 +34,31 @@ define void @packPHIs(ptr %ptr) {
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packPHIs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
br label %loop
@@ -63,14 +87,31 @@ define void @packFromOtherBB(ptr %ptr, i8 %val) {
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromOtherBB(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[VAL:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[VAL]], 0
+; TOPDOWN-NEXT: [[MUL1:%.*]] = mul i8 [[VAL]], 1
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %val, 0
%mul1 = mul i8 %val, 1
@@ -97,12 +138,25 @@ define void @packFromDiffBBs(ptr %ptr, i8 %v) {
; CHECK-NEXT: br label %[[BB:.*]]
; CHECK: [[BB]]:
; CHECK-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromDiffBBs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[V:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*:]]
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[V]], 1
+; TOPDOWN-NEXT: br label %[[BB:.*]]
+; TOPDOWN: [[BB]]:
+; TOPDOWN-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %v, 1
br label %bb
@@ -121,3 +175,8 @@ bb:
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
new file mode 100644
index 0000000000000..f41e1c6e3da82
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -0,0 +1,441 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 \
+; RUN: -sbvec-collect-seeds=loads \
+; RUN: -sbvec-passes="seed-collection<tr-save,bottom-up-vec(top-down),tr-accept>" \
+; RUN: %s -S | FileCheck %s
+
+; Tests: successful bundle match (baseline) and !IMaps->isVectorized(UI0) on
+; the outer loop's second use edge (%ld0 used twice by the same fadd).
+define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_fadd_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META0]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META0]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define void @load_chain_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_chain_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fmul <2 x float> [[VECL]], splat (float 3.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[VEC1:%.*]] = fadd <2 x float> [[VEC]], splat (float 2.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC1]], ptr [[PTR2_0]], align 4, !sandboxvec [[META1]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fmul0 = fmul float %ld0, 3.0
+ %fmul1 = fmul float %ld1, 3.0
+
+ %fadd0 = fadd float %fmul0, 2.0
+ %fadd1 = fadd float %fmul1, 2.0
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define float @load_fadd_external_use(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define float @load_fadd_external_use(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META2]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META2]]
+; CHECK-NEXT: ret float [[UNPACK]]
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+
+ ret float %fadd0
+}
+
+; Both lanes feed the *same* user instruction (once per operand). The user
+; bundle must not be formed out of duplicate instructions, so the fadd stays
+; scalar while the loads still widen.
+define float @single_user_no_duplicate(ptr %ptr) {
+; CHECK-LABEL: define float @single_user_no_duplicate(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META3]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META3]]
+; CHECK-NEXT: [[FADD:%.*]] = fadd float [[UNPACK]], [[UNPACK1]]
+; CHECK-NEXT: ret float [[FADD]]
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd = fadd float %ld0, %ld1
+ ret float %fadd
+}
+
+; The candidate users live in different blocks from each other, so they must
+; not be bundled together.
+define void @users_in_different_blocks(ptr %ptr, ptr %ptr2, i1 %c) {
+; CHECK-LABEL: define void @users_in_different_blocks(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META4:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META4]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META4]]
+; CHECK-NEXT: br i1 [[C]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK: [[BB0]]:
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[BB1]]:
+; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+ br i1 %c, label %if.then, label %if.else
+
+if.then:
+ %fadd0 = fadd float %ld0, %ld0
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ store float %fadd0, ptr %ptr2_0
+ ret void
+
+if.else:
+ %fadd1 = fadd float %ld1, %ld1
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+; Lane 0 feeds fadd, lane 1 feeds fmul — opcode mismatch rejects the bundle.
+define void @user_opcode_mismatch(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_opcode_mismatch(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META5:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META5]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META5]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FMUL1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fmul1 = fmul float %ld1, %ld1
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store float %fmul1, ptr %ptr2_1, align 4
+ ret void
+}
+
+; Lane 0's user is fadd float, lane 1's user is fadd double — type mismatch.
+define void @user_type_mismatch(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_type_mismatch(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr double, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META6:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META6]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META6]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[EXT1:%.*]] = fpext float [[UNPACK1]] to double
+; CHECK-NEXT: [[FADD1:%.*]] = fadd double [[EXT1]], [[EXT1]]
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store double [[FADD1]], ptr [[PTR2_1]], align 8
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr double, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd float %ld0, %ld0
+ %ext1 = fpext float %ld1 to double
+ %fadd1 = fadd double %ext1, %ext1
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store double %fadd1, ptr %ptr2_1, align 8
+ ret void
+}
+
+; Lane 0 uses ld0 at operand 0; lane 1 uses ld1 at operand 1 (fsub is not
+; commutative). Operand-index mismatch rejects the bundle.
+define void @user_operand_index_mismatch(ptr %ptr, ptr %ptr2, float %x) {
+; CHECK-LABEL: define void @user_operand_index_mismatch(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[X:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META7:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META7]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META7]]
+; CHECK-NEXT: [[FSUB0:%.*]] = fsub float [[UNPACK]], [[X]]
+; CHECK-NEXT: [[FSUB1:%.*]] = fsub float [[X]], [[UNPACK1]]
+; CHECK-NEXT: store float [[FSUB0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FSUB1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fsub0 = fsub float %ld0, %x
+ %fsub1 = fsub float %x, %ld1
+
+ store float %fsub0, ptr %ptr2_0, align 4
+ store float %fsub1, ptr %ptr2_1, align 4
+ ret void
+}
+
+; ld0 has two fadd users; the first bundle {fadd0,fadd1} vectorizes fadd1.
+; When matching the second bundle, fadd1 is skipped (already vectorized) and
+; fadd1b is chosen instead.
+define void @user_already_vectorized(ptr %ptr, ptr %ptr2, float %a, float %b) {
+; CHECK-LABEL: define void @user_already_vectorized(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META8:![0-9]+]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META8]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META8]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[A]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK2]], !sandboxvec [[META8]]
+; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[A]]
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_2]], align 4, !sandboxvec [[META8]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+ %ptr2_3 = getelementptr float, ptr %ptr2, i32 3
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd float %ld0, %a
+ %fadd0b = fadd float %ld0, %b
+ %fadd1 = fadd float %ld1, %a
+ %fadd1b = fadd float %ld1, %b
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store float %fadd1, ptr %ptr2_1, align 4
+ store float %fadd0b, ptr %ptr2_2, align 4
+ store float %fadd1b, ptr %ptr2_3, align 4
+ ret void
+}
+; The fadd user bundle passes the getNextUserBundles() checks (same opcode,
+; type, BB, operand index) but legality returns Pack due to different
+; fast-math flags. The recursion must stop there: loads widen, fadds stay
+; scalar and are fed by unpacks.
+define void @user_diff_fast_math_flags(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_diff_fast_math_flags(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META9:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META9]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META9]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd fast float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store float %fadd1, ptr %ptr2_1, align 4
+ ret void
+}
+
+; Same as above but the user bundle packs due to different wrap flags
+; (add nsw vs add).
+define void @user_diff_wrap_flags(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_diff_wrap_flags(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr i32, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr i32, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 4, !sandboxvec [[META10:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i32> [[VECL]], i32 0, !sandboxvec [[META10]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i32> [[VECL]], i32 1, !sandboxvec [[META10]]
+; CHECK-NEXT: [[ADD0:%.*]] = add nsw i32 [[UNPACK]], 1
+; CHECK-NEXT: [[ADD1:%.*]] = add i32 [[UNPACK1]], 1
+; CHECK-NEXT: store i32 [[ADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store i32 [[ADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i32, ptr %ptr, i32 0
+ %ptr1 = getelementptr i32, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr i32, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr i32, ptr %ptr2, i32 1
+
+ %ld0 = load i32, ptr %ptr0, align 4
+ %ld1 = load i32, ptr %ptr1, align 4
+
+ %add0 = add nsw i32 %ld0, 1
+ %add1 = add i32 %ld1, 1
+
+ store i32 %add0, ptr %ptr2_0, align 4
+ store i32 %add1, ptr %ptr2_1, align 4
+ ret void
+}
+
+; Lane 0's value is used twice in %fadd0 (operands {0,1}) but lane 1's matching
+; user only uses %ld1 once (at operand 0). The operand-usage patterns differ, so
+; getNextUserBundles() rejects the bundle: the loads widen while the fadds stay
+; scalar and are fed by unpacks.
+define void @user_duplicate_operand_other(ptr %ptr, ptr %ptr2, float %other) {
+; CHECK-LABEL: define void @user_duplicate_operand_other(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[OTHER:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META11:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META11]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META11]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[OTHER]]
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %other
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store float %fadd1, ptr %ptr2_1, align 4
+ ret void
+}
+
+; The store user bundle forms but legality packs it because the stores are
+; not consecutive (there is a gap in the destination).
+define void @user_stores_not_consecutive(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_stores_not_consecutive(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META12:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META12]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META12]]
+; CHECK-NEXT: store float [[UNPACK]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[UNPACK1]], ptr [[PTR2_2]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ store float %ld0, ptr %ptr2_0, align 4
+ store float %ld1, ptr %ptr2_2, align 4
+ ret void
+}
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+; CHECK: [[META2]] = distinct !{!"sandboxregion"}
+; CHECK: [[META3]] = distinct !{!"sandboxregion"}
+; CHECK: [[META4]] = distinct !{!"sandboxregion"}
+; CHECK: [[META5]] = distinct !{!"sandboxregion"}
+; CHECK: [[META6]] = distinct !{!"sandboxregion"}
+; CHECK: [[META7]] = distinct !{!"sandboxregion"}
+; CHECK: [[META8]] = distinct !{!"sandboxregion"}
+; CHECK: [[META9]] = distinct !{!"sandboxregion"}
+; CHECK: [[META10]] = distinct !{!"sandboxregion"}
+; CHECK: [[META11]] = distinct !{!"sandboxregion"}
+; CHECK: [[META12]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
index ea6cf7d45f525..84b8812ca0db3 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -18,9 +18,11 @@
#include "llvm/IR/Dominators.h"
#include "llvm/SandboxIR/Context.h"
#include "llvm/SandboxIR/Function.h"
+#include "llvm/SandboxIR/Instruction.h"
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Type.h"
#include "llvm/Support/SourceMgr.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
@@ -805,3 +807,230 @@ define void @foo(i32 %s0, <4 x i32> %v0, i32 %s1, <2 x i32> %v1, <3 x i32> %v2,
EXPECT_EQ(Elms, Bndl);
EXPECT_THAT(Lanes, testing::ElementsAre(0, 1, 5, 6, 8, 11));
}
+
+TEST_F(VecUtilsTest, GetNextUserBundle) {
+ parseIR(R"IR(
+define void @match(ptr %p) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld0
+ %add1 = fadd float %ld1, %ld1
+ ret void
+}
+
+define void @opcode_mismatch(ptr %p) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld0
+ %sub1 = fsub float %ld1, %ld1
+ ret void
+}
+
+define void @type_mismatch(ptr %pf, ptr %pd) {
+entry:
+ %f0 = load float, ptr %pf
+ %d1 = load double, ptr %pd
+ %add0 = fadd float %f0, %f0
+ %add1 = fadd double %d1, %d1
+ ret void
+}
+
+define void @block_mismatch(ptr %p) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld0
+ br label %bb1
+bb1:
+ %add1 = fadd float %ld1, %ld1
+ ret void
+}
+
+define void @operand_mismatch(ptr %p) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld1
+ %add1 = fadd float %ld0, %ld1
+ ret void
+}
+
+define void @duplicate_operand_mismatch(ptr %p, float %other) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld0
+ %add1 = fadd float %ld1, %other
+ ret void
+}
+
+define void @missing_lane_user(ptr %p) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld0
+ ret void
+}
+
+define void @vectorized_user(ptr %p) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld0
+ %add1 = fadd float %ld1, %ld1
+ ret void
+}
+
+define void @vectorized_seed_user(ptr %p) {
+entry:
+ %gep0 = getelementptr float, ptr %p, i32 0
+ %gep1 = getelementptr float, ptr %p, i32 1
+ %ld0 = load float, ptr %gep0
+ %ld1 = load float, ptr %gep1
+ %add0 = fadd float %ld0, %ld0
+ %add1 = fadd float %ld1, %ld1
+ ret void
+}
+)IR");
+
+ auto withFunction = [this](StringRef FuncName, auto &&TestFn) {
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction(FuncName));
+ TestFn(*F, IMaps);
+ };
+
+ withFunction("match",
+ [](sandboxir::Function &F, sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ auto *Add0 = &*It++;
+ auto *Add1 = &*It++;
+
+ ASSERT_EQ(Add0->getOperand(0), Ld0);
+ ASSERT_EQ(Add1->getOperand(0), Ld1);
+
+ auto NextUserBundles =
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps);
+ ASSERT_EQ(NextUserBundles.size(), 1u);
+ ASSERT_EQ(NextUserBundles[0].size(), 2u);
+ EXPECT_EQ(NextUserBundles[0][0], Add0);
+ EXPECT_EQ(NextUserBundles[0][1], Add1);
+ });
+
+ withFunction("opcode_mismatch", [](sandboxir::Function &F,
+ sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ });
+
+ withFunction(
+ "type_mismatch", [](sandboxir::Function &F, sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ auto *F0 = &*It++;
+ auto *D1 = &*It++;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({F0, D1}, IMaps).empty());
+ });
+
+ withFunction("block_mismatch", [](sandboxir::Function &F,
+ sandboxir::InstrMaps &IMaps) {
+ auto &Entry = getBasicBlockByName(F, "entry");
+ auto It = Entry.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ });
+
+ withFunction("operand_mismatch", [](sandboxir::Function &F,
+ sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ });
+
+ withFunction("duplicate_operand_mismatch", [](sandboxir::Function &F,
+ sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ // Lane 0's user uses Ld0 at operands {0, 1} but lane 1's user
+ // uses Ld1 only at operand {0}, so no bundle should form.
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ });
+
+ withFunction("missing_lane_user", [](sandboxir::Function &F,
+ sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ });
+
+ withFunction("vectorized_user", [](sandboxir::Function &F,
+ sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ auto *Add0 = &*It++;
+ auto *Add1 = &*It++;
+ (void)Add0;
+ sandboxir::Action A(nullptr, {Add1}, {}, 0);
+ IMaps.registerVector({Add1}, &A);
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ });
+
+ withFunction("vectorized_seed_user", [](sandboxir::Function &F,
+ sandboxir::InstrMaps &IMaps) {
+ auto &BB = getBasicBlockByName(F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ auto *Add0 = &*It++;
+ sandboxir::Action A(nullptr, {Add0}, {}, 0);
+ IMaps.registerVector({Add0}, &A);
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ });
+}
>From ce54474efae1476b43478da359493cdf5fd4f4d8 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Tue, 21 Jul 2026 10:53:30 -0500
Subject: [PATCH 03/11] [SBVec] Refactor BottomUpVec pass for clarity and
maintainability
- Corrected comments to clarify the direction of def-use and use-def chains.
- Changed the initialization of the SchedDirection variable to improve clarity.
- Updated documentation in vectorizeRec() to better describe the purpose of UserBndl.
- Removed outdated TODO comment regarding top-down vectorization scheduling.
---
.../Vectorize/SandboxVectorizer/Passes/BottomUpVec.h | 9 +++++----
.../Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp | 7 +------
2 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index bdddc38906023..fec4192f7626e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -30,7 +30,7 @@ namespace llvm::sandboxir {
/// The "seed slice" is a vector of instructions that can be used as a starting
/// point for vectorization, like stores (loads) to consecutive memory
/// addresses. Starting from the seed instructions, it walks up (down) the
-/// def-use (use-def) chains looking for more instructions that can be
+/// use-def (def-use) chains looking for more instructions that can be
/// vectorized. This pass will generate vector code if it can legally vectorize
/// the code, regardless of whether it is profitable or not. For now
/// profitability is checked at the end of the region pass pipeline by a
@@ -43,7 +43,7 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
static constexpr StringRef TopDownArgStr = "top-down";
static constexpr StringRef BottomUpArgStr = "bottom-up";
/// Direction for vectorization, defaults to bottom-up.
- SchedDirection Dir = SchedDirection::BottomUp;
+ SchedDirection Dir;
/// The original instructions that are potentially dead after vectorization.
DenseSet<Instruction *> DeadInstrCandidates;
/// Maps scalars to vectors.
@@ -94,8 +94,8 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
/// vectorize in vectorizeRec().
unsigned DebugBndlCnt = 0;
- /// Recursively try to vectorize \p Bndl. For bottom-up vectorization \p
- /// UserBndl tracks the bundle of users that led to this recursion.
+ /// Recursively try to vectorize \p Bndl. \p UserBndl identifies the
+ /// users that this recursive call originates from.
Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
unsigned Depth, LegalityAnalysis &Legality);
/// If the values in \p Bndl have external users, then emit unpacks and
@@ -109,6 +109,7 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
public:
BottomUpVec(StringRef AuxArg) : RegionPass("bottom-up-vec") {
+ /// TODO: Drop the AuxArg.empty() part
if (AuxArg.empty() || AuxArg == BottomUpArgStr) {
Dir = SchedDirection::BottomUp;
} else if (AuxArg == TopDownArgStr) {
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index e7599035cc3a6..b8e99b0ccf67c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -286,7 +286,6 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
DebugBndlCnt++ >= StopBundle && StopBundle != StopBundleDisabled;
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
VecUtils::dump(Bndl));
- /// TODO: Enable scheduling for topdown vectorization
const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
: Legality.canVectorize(Bndl);
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
@@ -294,11 +293,7 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
if (Dir == SchedDirection::TopDown) {
// A non-Widen result means we can't extend the vectorized region into
// this bundle, so leave its instructions scalar and don't record an
- // action for it. The scalar users of the already-widened defs get their
- // values through the unpacks emitted by emitUnpacksForExternalUses().
- // Note: The DiamondReuse* results are unreachable in the top-down
- // direction because getNextUserBundles() skips already-vectorized users,
- // so a bundle never contains instructions registered in IMaps.
+ // action for it.
if (LegalityRes.getSubclassID() != LegalityResultID::Widen)
return nullptr;
>From c02362f0ebf14c6c38a2a6b7a797138c97df843e Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Tue, 21 Jul 2026 13:28:36 -0500
Subject: [PATCH 04/11] [SBVec] Track claimed users across bundles
---
.../Vectorize/SandboxVectorizer/VecUtils.cpp | 16 ++++--
.../SandboxVectorizer/topdown_vec.ll | 51 ++++++++++++++-----
2 files changed, 51 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
index 9f5b5422b9b13..672626cf03fff 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
@@ -33,13 +33,14 @@ SmallVector<BundleTy> VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl,
Value *V0 = Bndl[0];
DenseSet<User *> SeenUsers;
+ SmallPtrSet<Instruction *, 4> Claimed;
// For each user U0 of lane 0, try to form a bundle of matching users across
// all lanes.
for (User *U0 : V0->users()) {
if (!SeenUsers.insert(U0).second)
continue;
auto *UI0 = dyn_cast<Instruction>(U0);
- if (!UI0 || IMaps.isVectorized(UI0))
+ if (!UI0 || IMaps.isVectorized(UI0) || Claimed.contains(UI0))
continue;
// The operand indices at which lane 0's user U0 uses lane 0's value V0.
@@ -52,8 +53,10 @@ SmallVector<BundleTy> VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl,
// Find a distinct matching user for each of the remaining lanes.
BundleTy NextUserBndl;
NextUserBndl.push_back(UI0);
- SmallPtrSet<Instruction *, 4> Claimed;
+ // Tentatively claim UI0; roll back if a full bundle can't be formed.
+ SmallVector<Instruction *, 4> NewlyClaimed;
Claimed.insert(UI0);
+ NewlyClaimed.push_back(UI0);
for (Value *V : drop_begin(Bndl)) {
Instruction *Match = nullptr;
for (User *U : V->users()) {
@@ -80,11 +83,18 @@ SmallVector<BundleTy> VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl,
break;
}
Claimed.insert(Match);
+ NewlyClaimed.push_back(Match);
NextUserBndl.push_back(Match);
}
- if (NextUserBndl.size() == Bndl.size())
+ if (NextUserBndl.size() == Bndl.size()) {
Bundles.emplace_back(std::move(NextUserBndl));
+ } else {
+ // Failed to form a full bundle; release the instructions we tentatively
+ // claimed so they remain available for other lane-0 users.
+ for (Instruction *I : NewlyClaimed)
+ Claimed.erase(I);
+ }
}
return Bundles;
}
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
index f41e1c6e3da82..7d4916f839d3c 100644
--- a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -250,26 +250,25 @@ define void @user_operand_index_mismatch(ptr %ptr, ptr %ptr2, float %x) {
ret void
}
-; ld0 has two fadd users; the first bundle {fadd0,fadd1} vectorizes fadd1.
-; When matching the second bundle, fadd1 is skipped (already vectorized) and
-; fadd1b is chosen instead.
+; ld0/ld1 each have two fadd users: one adding %a and one adding %b. The two
+; user bundles must not fight over the same instruction: getNextUserBundles()
+; tracks claimed users across all bundles it forms, so once {fadd0b,fadd1b} is
+; claimed, the second bundle picks {fadd0,fadd1} rather than reusing fadd1b.
+; Both bundles widen, producing two fadd <2 x float>.
define void @user_already_vectorized(ptr %ptr, ptr %ptr2, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized(
; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META8:![0-9]+]]
-; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META8:![0-9]+]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META8]]
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META8]]
; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META8]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META8]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META8]]
-; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[A]]
-; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK2]], !sandboxvec [[META8]]
-; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[A]]
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META8]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META8]]
+; CHECK-NEXT: store <2 x float> [[VEC4]], ptr [[PTR2_0]], align 4, !sandboxvec [[META8]]
; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_2]], align 4, !sandboxvec [[META8]]
; CHECK-NEXT: ret void
;
@@ -424,6 +423,32 @@ define void @user_stores_not_consecutive(ptr %ptr, ptr %ptr2) {
store float %ld1, ptr %ptr2_2, align 4
ret void
}
+
+; FIXME: Following test needs diamond reuse multi-input support in topdown
+; vectorizer.
+;define void @user_diamond_reuse_multi_input(ptr %ptr, ptr %ptr2) {
+; %ptr0 = getelementptr float, ptr %ptr, i32 0
+; %ptr1 = getelementptr float, ptr %ptr, i32 1
+; %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+; %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+;
+; %ld0 = load float, ptr %ptr0, align 4
+; %ld1 = load float, ptr %ptr1, align 4
+;
+; %sub0 = fsub float %ld0, 0.000000e+00
+; %sub1 = fsub float %ld1, 1.000000e+00
+;
+; %add0 = fadd float %ld0, 0.000000e+00
+; %add1 = fadd float %ld1, 0.000000e+00
+;
+; %diam0 = fsub float %sub0, %add0
+; %diam1 = fsub float %sub1, %add1
+;
+; store float %diam0, ptr %ptr2_0, align 4
+; store float %diam1, ptr %ptr2_2, align 4
+; ret void
+;}
+
;.
; CHECK: [[META0]] = distinct !{!"sandboxregion"}
; CHECK: [[META1]] = distinct !{!"sandboxregion"}
>From 5b9e622ff8ba356330d80cfce77ecb42d54030f4 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Tue, 21 Jul 2026 14:08:26 -0500
Subject: [PATCH 05/11] 3 element tests
- nits
---
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 4 -
.../SandboxVectorizer/topdown_vec.ll | 135 +++++++++++++++++-
2 files changed, 130 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index b8e99b0ccf67c..b01ab29c2223c 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -306,10 +306,6 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
// Walk down the def-use chain. Each lane in \p Bndl may feed several
// users, so we form every compatible user bundle and recurse into each
// one.
- //
- // Recursing right after forming each bundle marks its instructions as
- // vectorized (pre-order registration), which prevents sibling bundles
- // from claiming the same instruction and guarantees termination.
for (const auto &NextUserBndl : VecUtils::getNextUserBundles(Bndl, *IMaps))
vectorizeRec(NextUserBndl, Bndl, Depth + 1, Legality);
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
index 7d4916f839d3c..f841b443817b9 100644
--- a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -1,8 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 \
-; RUN: -sbvec-collect-seeds=loads \
-; RUN: -sbvec-passes="seed-collection<tr-save,bottom-up-vec(top-down),tr-accept>" \
-; RUN: %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-collect-seeds=loads -sbvec-passes="seed-collection<tr-save,bottom-up-vec(top-down),tr-accept>" %s -S | FileCheck %s
; Tests: successful bundle match (baseline) and !IMaps->isVectorized(UI0) on
; the outer loop's second use edge (%ld0 used twice by the same fadd).
@@ -283,8 +280,8 @@ define void @user_already_vectorized(ptr %ptr, ptr %ptr2, float %a, float %b) {
%ld1 = load float, ptr %ptr1, align 4
%fadd0 = fadd float %ld0, %a
- %fadd0b = fadd float %ld0, %b
%fadd1 = fadd float %ld1, %a
+ %fadd0b = fadd float %ld0, %b
%fadd1b = fadd float %ld1, %b
store float %fadd0, ptr %ptr2_0, align 4
@@ -449,6 +446,131 @@ define void @user_stores_not_consecutive(ptr %ptr, ptr %ptr2) {
; ret void
;}
+; 3-wide baseline: three lanes widen through load -> fadd -> store.
+define void @load_fadd_store_3wide(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_fadd_store_3wide(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META13:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[VECL]], !sandboxvec [[META13]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META13]]
+; CHECK-NEXT: ret void
+;
+ %g0 = getelementptr float, ptr %ptr, i32 0
+ %g1 = getelementptr float, ptr %ptr, i32 1
+ %g2 = getelementptr float, ptr %ptr, i32 2
+ %ld0 = load float, ptr %g0
+ %ld1 = load float, ptr %g1
+ %ld2 = load float, ptr %g2
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+ %fadd2 = fadd float %ld2, %ld2
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ store float %fadd2, ptr %ptr2_2
+ ret void
+}
+
+; 3-wide version of @user_already_vectorized: each of the three loads feeds two
+; fadd users (one adding %a, one adding %b). getNextUserBundles() must track
+; claimed users across all bundles it forms so the two 3-lane bundles pick
+; disjoint instructions, producing two fadd <3 x float>.
+define void @user_already_vectorized_3wide(ptr %ptr, ptr %ptr2, float %a, float %b) {
+; CHECK-LABEL: define void @user_already_vectorized_3wide(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <3 x float> poison, float [[A]], i32 0, !sandboxvec [[META14:![0-9]+]]
+; CHECK-NEXT: [[PACK4:%.*]] = insertelement <3 x float> [[PACK3]], float [[A]], i32 1, !sandboxvec [[META14]]
+; CHECK-NEXT: [[PACK5:%.*]] = insertelement <3 x float> [[PACK4]], float [[A]], i32 2, !sandboxvec [[META14]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META14]]
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META14]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <3 x float> [[PACK1]], float [[B]], i32 2, !sandboxvec [[META14]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META14]]
+; CHECK-NEXT: [[VEC6:%.*]] = fadd <3 x float> [[VECL]], [[PACK5]], !sandboxvec [[META14]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK2]], !sandboxvec [[META14]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_3:%.*]] = getelementptr float, ptr [[PTR2]], i32 3
+; CHECK-NEXT: store <3 x float> [[VEC6]], ptr [[PTR2_0]], align 4, !sandboxvec [[META14]]
+; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR2_3]], align 4, !sandboxvec [[META14]]
+; CHECK-NEXT: ret void
+;
+ %g0 = getelementptr float, ptr %ptr, i32 0
+ %g1 = getelementptr float, ptr %ptr, i32 1
+ %g2 = getelementptr float, ptr %ptr, i32 2
+ %ld0 = load float, ptr %g0
+ %ld1 = load float, ptr %g1
+ %ld2 = load float, ptr %g2
+
+ %fadd0a = fadd float %ld0, %a
+ %fadd0b = fadd float %ld0, %b
+ %fadd1a = fadd float %ld1, %a
+ %fadd1b = fadd float %ld1, %b
+ %fadd2a = fadd float %ld2, %a
+ %fadd2b = fadd float %ld2, %b
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+ %ptr2_3 = getelementptr float, ptr %ptr2, i32 3
+ %ptr2_4 = getelementptr float, ptr %ptr2, i32 4
+ %ptr2_5 = getelementptr float, ptr %ptr2, i32 5
+ store float %fadd0a, ptr %ptr2_0
+ store float %fadd1a, ptr %ptr2_1
+ store float %fadd2a, ptr %ptr2_2
+ store float %fadd0b, ptr %ptr2_3
+ store float %fadd1b, ptr %ptr2_4
+ store float %fadd2b, ptr %ptr2_5
+ ret void
+}
+
+; 3-wide rejection: lanes 0 and 1 feed fadd but lane 2 feeds fmul, so the user
+; bundle can't be formed. The loads still widen to <3 x float> and the
+; arithmetic stays scalar, fed by unpacks.
+define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_opcode_mismatch_3wide(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META15:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META15]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META15]]
+; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META15]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
+; CHECK-NEXT: [[FMUL2:%.*]] = fmul float [[UNPACK2]], [[UNPACK2]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: store float [[FMUL2]], ptr [[PTR2_2]], align 4
+; CHECK-NEXT: ret void
+;
+ %g0 = getelementptr float, ptr %ptr, i32 0
+ %g1 = getelementptr float, ptr %ptr, i32 1
+ %g2 = getelementptr float, ptr %ptr, i32 2
+ %ld0 = load float, ptr %g0
+ %ld1 = load float, ptr %g1
+ %ld2 = load float, ptr %g2
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+ %fmul2 = fmul float %ld2, %ld2
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ store float %fmul2, ptr %ptr2_2
+ ret void
+}
+
;.
; CHECK: [[META0]] = distinct !{!"sandboxregion"}
; CHECK: [[META1]] = distinct !{!"sandboxregion"}
@@ -463,4 +585,7 @@ define void @user_stores_not_consecutive(ptr %ptr, ptr %ptr2) {
; CHECK: [[META10]] = distinct !{!"sandboxregion"}
; CHECK: [[META11]] = distinct !{!"sandboxregion"}
; CHECK: [[META12]] = distinct !{!"sandboxregion"}
+; CHECK: [[META13]] = distinct !{!"sandboxregion"}
+; CHECK: [[META14]] = distinct !{!"sandboxregion"}
+; CHECK: [[META15]] = distinct !{!"sandboxregion"}
;.
>From 30f0db66f1985a6e5679d12fa331beb4f2b2ee7e Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Tue, 21 Jul 2026 18:38:32 -0500
Subject: [PATCH 06/11] Add 3-way test to check for consecutive matching
---
.../SandboxVectorizer/topdown_vec.ll | 45 +++++++++++++++++++
1 file changed, 45 insertions(+)
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
index f841b443817b9..a9a8ef077523f 100644
--- a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -571,6 +571,50 @@ define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
ret void
}
+; 3-wide non-consecutive rejection: lane 0 (fadd) and lane 2 (fadd) have
+; matching users but the middle lane 1 uses fmul. getNextUserBundles() matches
+; lanes consecutively, so it must NOT skip the gap at lane 1 and pair lane 0's
+; user with lane 2's user (which would require a shuffle). The bundle is
+; rejected: loads widen to <3 x float> and all three ops stay scalar.
+define void @user_middle_lane_mismatch_3wide(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_middle_lane_mismatch_3wide(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META16:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META16]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META16]]
+; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META16]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
+; CHECK-NEXT: [[FADD2:%.*]] = fadd float [[UNPACK2]], [[UNPACK2]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FMUL1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: store float [[FADD2]], ptr [[PTR2_2]], align 4
+; CHECK-NEXT: ret void
+;
+ %g0 = getelementptr float, ptr %ptr, i32 0
+ %g1 = getelementptr float, ptr %ptr, i32 1
+ %g2 = getelementptr float, ptr %ptr, i32 2
+ %ld0 = load float, ptr %g0
+ %ld1 = load float, ptr %g1
+ %ld2 = load float, ptr %g2
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fmul1 = fmul float %ld1, %ld1
+ %fadd2 = fadd float %ld2, %ld2
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+ store float %fadd0, ptr %ptr2_0
+ store float %fmul1, ptr %ptr2_1
+ store float %fadd2, ptr %ptr2_2
+ ret void
+}
+
;.
; CHECK: [[META0]] = distinct !{!"sandboxregion"}
; CHECK: [[META1]] = distinct !{!"sandboxregion"}
@@ -588,4 +632,5 @@ define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
; CHECK: [[META13]] = distinct !{!"sandboxregion"}
; CHECK: [[META14]] = distinct !{!"sandboxregion"}
; CHECK: [[META15]] = distinct !{!"sandboxregion"}
+; CHECK: [[META16]] = distinct !{!"sandboxregion"}
;.
>From 39aca64e8e6ad5675605489b0717d69baa8eac4d Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Thu, 23 Jul 2026 18:20:32 -0500
Subject: [PATCH 07/11] Pass Claimed by ref to getNextUserBundles
A user should not be claimed by multiple successful
bundles. Added a test for this.
---
.../Vectorize/SandboxVectorizer/VecUtils.h | 3 +-
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 4 +-
.../Vectorize/SandboxVectorizer/VecUtils.cpp | 6 +-
.../SandboxVectorizer/topdown_vec.ll | 67 +++++++++++++++++--
4 files changed, 68 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 8f37d2bafbd79..2504174591993 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -232,7 +232,8 @@ class VecUtils {
/// For each user of lane 0 in \p Bndl, try to form a bundle of matching
/// users for all lanes. Returns all complete user bundles found.
LLVM_ABI static SmallVector<BundleTy>
- getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps);
+ getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
+ SmallPtrSet<Instruction *, 4> &Claimed);
/// Helper struct for `matchPack()`. Describes the instructions and operands
/// of a pack pattern.
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index b01ab29c2223c..77d47b6bbc1e7 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -306,7 +306,9 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
// Walk down the def-use chain. Each lane in \p Bndl may feed several
// users, so we form every compatible user bundle and recurse into each
// one.
- for (const auto &NextUserBndl : VecUtils::getNextUserBundles(Bndl, *IMaps))
+ SmallPtrSet<Instruction *, 4> Claimed;
+ for (const auto &NextUserBndl :
+ VecUtils::getNextUserBundles(Bndl, *IMaps, Claimed))
vectorizeRec(NextUserBndl, Bndl, Depth + 1, Legality);
return Action;
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
index 672626cf03fff..29c8edbcc2427 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
@@ -15,8 +15,9 @@
namespace llvm::sandboxir {
-SmallVector<BundleTy> VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl,
- const InstrMaps &IMaps) {
+SmallVector<BundleTy>
+VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
+ SmallPtrSet<Instruction *, 4> &Claimed) {
SmallVector<BundleTy> Bundles;
if (Bndl.empty())
return Bundles;
@@ -33,7 +34,6 @@ SmallVector<BundleTy> VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl,
Value *V0 = Bndl[0];
DenseSet<User *> SeenUsers;
- SmallPtrSet<Instruction *, 4> Claimed;
// For each user U0 of lane 0, try to form a bundle of matching users across
// all lanes.
for (User *U0 : V0->users()) {
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
index a9a8ef077523f..a023989909562 100644
--- a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -248,9 +248,9 @@ define void @user_operand_index_mismatch(ptr %ptr, ptr %ptr2, float %x) {
}
; ld0/ld1 each have two fadd users: one adding %a and one adding %b. The two
-; user bundles must not fight over the same instruction: getNextUserBundles()
-; tracks claimed users across all bundles it forms, so once {fadd0b,fadd1b} is
-; claimed, the second bundle picks {fadd0,fadd1} rather than reusing fadd1b.
+; user bundles must not fight over the same instruction: BottomUpVec passes a
+; Claimed set into getNextUserBundles(), so once {fadd0b,fadd1b} is claimed,
+; the second bundle picks {fadd0,fadd1} rather than reusing fadd1b.
; Both bundles widen, producing two fadd <2 x float>.
define void @user_already_vectorized(ptr %ptr, ptr %ptr2, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized(
@@ -571,6 +571,58 @@ define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
ret void
}
+; 3-wide Claimed rollback: each load has + %a and + %b fadd users, but ld2 has
+; no + %a user. BottomUpVec passes a Claimed set into getNextUserBundles(); a
+; failed {fadd0a,fadd1a,?} attempt tentatively claims fadd0a/fadd1a and must roll
+; them back so {fadd0b,fadd1b,fadd2b} can still form. Loads widen; the + %b
+; chain vectorizes while the + %a pair stays scalar.
+define void @claimed_rollback_partial_bundle(ptr %ptr, ptr %ptr2, float %a, float %b) {
+; CHECK-LABEL: define void @claimed_rollback_partial_bundle(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META16:![0-9]+]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META16]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <3 x float> [[PACK2]], float [[B]], i32 2, !sandboxvec [[META16]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META16]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META16]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META16]]
+; CHECK-NEXT: [[FADD0A:%.*]] = fadd float [[UNPACK]], [[A]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK3]], !sandboxvec [[META16]]
+; CHECK-NEXT: [[FADD1A:%.*]] = fadd float [[UNPACK1]], [[A]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
+; CHECK-NEXT: store float [[FADD0A]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FADD1A]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR2_2]], align 4, !sandboxvec [[META16]]
+; CHECK-NEXT: ret void
+;
+ %g0 = getelementptr float, ptr %ptr, i32 0
+ %g1 = getelementptr float, ptr %ptr, i32 1
+ %g2 = getelementptr float, ptr %ptr, i32 2
+ %ld0 = load float, ptr %g0
+ %ld1 = load float, ptr %g1
+ %ld2 = load float, ptr %g2
+
+ %fadd0a = fadd float %ld0, %a
+ %fadd0b = fadd float %ld0, %b
+ %fadd1a = fadd float %ld1, %a
+ %fadd1b = fadd float %ld1, %b
+ %fadd2b = fadd float %ld2, %b
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+ %ptr2_3 = getelementptr float, ptr %ptr2, i32 3
+ %ptr2_4 = getelementptr float, ptr %ptr2, i32 4
+ store float %fadd0a, ptr %ptr2_0
+ store float %fadd1a, ptr %ptr2_1
+ store float %fadd0b, ptr %ptr2_2
+ store float %fadd1b, ptr %ptr2_3
+ store float %fadd2b, ptr %ptr2_4
+ ret void
+}
+
; 3-wide non-consecutive rejection: lane 0 (fadd) and lane 2 (fadd) have
; matching users but the middle lane 1 uses fmul. getNextUserBundles() matches
; lanes consecutively, so it must NOT skip the gap at lane 1 and pair lane 0's
@@ -580,10 +632,10 @@ define void @user_middle_lane_mismatch_3wide(ptr %ptr, ptr %ptr2) {
; CHECK-LABEL: define void @user_middle_lane_mismatch_3wide(
; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META16:![0-9]+]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META16]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META16]]
-; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META16]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META17:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META17]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META17]]
+; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META17]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT: [[FADD2:%.*]] = fadd float [[UNPACK2]], [[UNPACK2]]
@@ -633,4 +685,5 @@ define void @user_middle_lane_mismatch_3wide(ptr %ptr, ptr %ptr2) {
; CHECK: [[META14]] = distinct !{!"sandboxregion"}
; CHECK: [[META15]] = distinct !{!"sandboxregion"}
; CHECK: [[META16]] = distinct !{!"sandboxregion"}
+; CHECK: [[META17]] = distinct !{!"sandboxregion"}
;.
>From 1b7b2e7f4a91cfce1b7d248129f1d402e5cc1cd5 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Wed, 29 Jul 2026 14:14:04 -0400
Subject: [PATCH 08/11] Remove stores
---
.../SandboxVectorizer/topdown_vec.ll | 241 +++---------------
1 file changed, 34 insertions(+), 207 deletions(-)
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
index a023989909562..81b5eec014ee6 100644
--- a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -9,8 +9,6 @@ define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META0]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META0]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
@@ -21,10 +19,6 @@ define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
%fadd0 = fadd float %ld0, %ld0
%fadd1 = fadd float %ld1, %ld1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- store float %fadd0, ptr %ptr2_0
- store float %fadd1, ptr %ptr2_1
ret void
}
@@ -35,8 +29,6 @@ define void @load_chain_store(ptr %ptr, ptr %ptr2) {
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
; CHECK-NEXT: [[VEC:%.*]] = fmul <2 x float> [[VECL]], splat (float 3.000000e+00), !sandboxvec [[META1]]
; CHECK-NEXT: [[VEC1:%.*]] = fadd <2 x float> [[VEC]], splat (float 2.000000e+00), !sandboxvec [[META1]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: store <2 x float> [[VEC1]], ptr [[PTR2_0]], align 4, !sandboxvec [[META1]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
@@ -50,22 +42,16 @@ define void @load_chain_store(ptr %ptr, ptr %ptr2) {
%fadd0 = fadd float %fmul0, 2.0
%fadd1 = fadd float %fmul1, 2.0
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- store float %fadd0, ptr %ptr2_0
- store float %fadd1, ptr %ptr2_1
ret void
}
-define float @load_fadd_external_use(ptr %ptr, ptr %ptr2) {
+define float @load_fadd_external_use(ptr %ptr) {
; CHECK-LABEL: define float @load_fadd_external_use(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META2:![0-9]+]]
; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META2]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META2]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META2]]
; CHECK-NEXT: ret float [[UNPACK]]
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
@@ -76,11 +62,6 @@ define float @load_fadd_external_use(ptr %ptr, ptr %ptr2) {
%fadd0 = fadd float %ld0, %ld0
%fadd1 = fadd float %ld1, %ld1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- store float %fadd0, ptr %ptr2_0
- store float %fadd1, ptr %ptr2_1
-
ret float %fadd0
}
@@ -108,9 +89,9 @@ define float @single_user_no_duplicate(ptr %ptr) {
; The candidate users live in different blocks from each other, so they must
; not be bundled together.
-define void @users_in_different_blocks(ptr %ptr, ptr %ptr2, i1 %c) {
+define void @users_in_different_blocks(ptr %ptr, i1 %c) {
; CHECK-LABEL: define void @users_in_different_blocks(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], i1 [[C:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]], i1 [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META4:![0-9]+]]
@@ -119,13 +100,9 @@ define void @users_in_different_blocks(ptr %ptr, ptr %ptr2, i1 %c) {
; CHECK-NEXT: br i1 [[C]], label %[[BB0:.*]], label %[[BB1:.*]]
; CHECK: [[BB0]]:
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
; CHECK-NEXT: ret void
; CHECK: [[BB1]]:
; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
-; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -137,70 +114,51 @@ entry:
if.then:
%fadd0 = fadd float %ld0, %ld0
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- store float %fadd0, ptr %ptr2_0
ret void
if.else:
%fadd1 = fadd float %ld1, %ld1
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- store float %fadd1, ptr %ptr2_1
ret void
}
; Lane 0 feeds fadd, lane 1 feeds fmul — opcode mismatch rejects the bundle.
-define void @user_opcode_mismatch(ptr %ptr, ptr %ptr2) {
+define void @user_opcode_mismatch(ptr %ptr) {
; CHECK-LABEL: define void @user_opcode_mismatch(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META5:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META5]]
; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META5]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FMUL1]], ptr [[PTR2_1]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
%ld0 = load float, ptr %ptr0, align 4
%ld1 = load float, ptr %ptr1, align 4
%fadd0 = fadd float %ld0, %ld0
%fmul1 = fmul float %ld1, %ld1
-
- store float %fadd0, ptr %ptr2_0, align 4
- store float %fmul1, ptr %ptr2_1, align 4
ret void
}
; Lane 0's user is fadd float, lane 1's user is fadd double — type mismatch.
-define void @user_type_mismatch(ptr %ptr, ptr %ptr2) {
+define void @user_type_mismatch(ptr %ptr) {
; CHECK-LABEL: define void @user_type_mismatch(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr double, ptr [[PTR2]], i32 1
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META6:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META6]]
; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META6]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[EXT1:%.*]] = fpext float [[UNPACK1]] to double
; CHECK-NEXT: [[FADD1:%.*]] = fadd double [[EXT1]], [[EXT1]]
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store double [[FADD1]], ptr [[PTR2_1]], align 8
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr double, ptr %ptr2, i32 1
%ld0 = load float, ptr %ptr0, align 4
%ld1 = load float, ptr %ptr1, align 4
@@ -208,42 +166,30 @@ define void @user_type_mismatch(ptr %ptr, ptr %ptr2) {
%fadd0 = fadd float %ld0, %ld0
%ext1 = fpext float %ld1 to double
%fadd1 = fadd double %ext1, %ext1
-
- store float %fadd0, ptr %ptr2_0, align 4
- store double %fadd1, ptr %ptr2_1, align 8
ret void
}
; Lane 0 uses ld0 at operand 0; lane 1 uses ld1 at operand 1 (fsub is not
; commutative). Operand-index mismatch rejects the bundle.
-define void @user_operand_index_mismatch(ptr %ptr, ptr %ptr2, float %x) {
+define void @user_operand_index_mismatch(ptr %ptr, float %x) {
; CHECK-LABEL: define void @user_operand_index_mismatch(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[X:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]], float [[X:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META7:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META7]]
; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META7]]
; CHECK-NEXT: [[FSUB0:%.*]] = fsub float [[UNPACK]], [[X]]
; CHECK-NEXT: [[FSUB1:%.*]] = fsub float [[X]], [[UNPACK1]]
-; CHECK-NEXT: store float [[FSUB0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FSUB1]], ptr [[PTR2_1]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
%ld0 = load float, ptr %ptr0, align 4
%ld1 = load float, ptr %ptr1, align 4
%fsub0 = fsub float %ld0, %x
%fsub1 = fsub float %x, %ld1
-
- store float %fsub0, ptr %ptr2_0, align 4
- store float %fsub1, ptr %ptr2_1, align 4
ret void
}
@@ -252,29 +198,21 @@ define void @user_operand_index_mismatch(ptr %ptr, ptr %ptr2, float %x) {
; Claimed set into getNextUserBundles(), so once {fadd0b,fadd1b} is claimed,
; the second bundle picks {fadd0,fadd1} rather than reusing fadd1b.
; Both bundles widen, producing two fadd <2 x float>.
-define void @user_already_vectorized(ptr %ptr, ptr %ptr2, float %a, float %b) {
+define void @user_already_vectorized(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META8:![0-9]+]]
; CHECK-NEXT: [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META8]]
; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META8]]
; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META8]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META8]]
; CHECK-NEXT: [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META8]]
; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META8]]
-; CHECK-NEXT: store <2 x float> [[VEC4]], ptr [[PTR2_0]], align 4, !sandboxvec [[META8]]
-; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_2]], align 4, !sandboxvec [[META8]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
- %ptr2_3 = getelementptr float, ptr %ptr2, i32 3
%ld0 = load float, ptr %ptr0, align 4
%ld1 = load float, ptr %ptr1, align 4
@@ -284,77 +222,55 @@ define void @user_already_vectorized(ptr %ptr, ptr %ptr2, float %a, float %b) {
%fadd0b = fadd float %ld0, %b
%fadd1b = fadd float %ld1, %b
- store float %fadd0, ptr %ptr2_0, align 4
- store float %fadd1, ptr %ptr2_1, align 4
- store float %fadd0b, ptr %ptr2_2, align 4
- store float %fadd1b, ptr %ptr2_3, align 4
ret void
}
; The fadd user bundle passes the getNextUserBundles() checks (same opcode,
; type, BB, operand index) but legality returns Pack due to different
; fast-math flags. The recursion must stop there: loads widen, fadds stay
; scalar and are fed by unpacks.
-define void @user_diff_fast_math_flags(ptr %ptr, ptr %ptr2) {
+define void @user_diff_fast_math_flags(ptr %ptr) {
; CHECK-LABEL: define void @user_diff_fast_math_flags(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META9:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META9]]
; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META9]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
%ld0 = load float, ptr %ptr0, align 4
%ld1 = load float, ptr %ptr1, align 4
%fadd0 = fadd fast float %ld0, %ld0
%fadd1 = fadd float %ld1, %ld1
-
- store float %fadd0, ptr %ptr2_0, align 4
- store float %fadd1, ptr %ptr2_1, align 4
ret void
}
; Same as above but the user bundle packs due to different wrap flags
; (add nsw vs add).
-define void @user_diff_wrap_flags(ptr %ptr, ptr %ptr2) {
+define void @user_diff_wrap_flags(ptr %ptr) {
; CHECK-LABEL: define void @user_diff_wrap_flags(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr i32, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr i32, ptr [[PTR2]], i32 1
; CHECK-NEXT: [[VECL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 4, !sandboxvec [[META10:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i32> [[VECL]], i32 0, !sandboxvec [[META10]]
; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i32> [[VECL]], i32 1, !sandboxvec [[META10]]
; CHECK-NEXT: [[ADD0:%.*]] = add nsw i32 [[UNPACK]], 1
; CHECK-NEXT: [[ADD1:%.*]] = add i32 [[UNPACK1]], 1
-; CHECK-NEXT: store i32 [[ADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store i32 [[ADD1]], ptr [[PTR2_1]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i32, ptr %ptr, i32 0
%ptr1 = getelementptr i32, ptr %ptr, i32 1
- %ptr2_0 = getelementptr i32, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr i32, ptr %ptr2, i32 1
%ld0 = load i32, ptr %ptr0, align 4
%ld1 = load i32, ptr %ptr1, align 4
%add0 = add nsw i32 %ld0, 1
%add1 = add i32 %ld1, 1
-
- store i32 %add0, ptr %ptr2_0, align 4
- store i32 %add1, ptr %ptr2_1, align 4
ret void
}
@@ -362,62 +278,42 @@ define void @user_diff_wrap_flags(ptr %ptr, ptr %ptr2) {
; user only uses %ld1 once (at operand 0). The operand-usage patterns differ, so
; getNextUserBundles() rejects the bundle: the loads widen while the fadds stay
; scalar and are fed by unpacks.
-define void @user_duplicate_operand_other(ptr %ptr, ptr %ptr2, float %other) {
+define void @user_duplicate_operand_other(ptr %ptr, float %other) {
; CHECK-LABEL: define void @user_duplicate_operand_other(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[OTHER:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]], float [[OTHER:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META11:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META11]]
; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META11]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[OTHER]]
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
%ld0 = load float, ptr %ptr0, align 4
%ld1 = load float, ptr %ptr1, align 4
%fadd0 = fadd float %ld0, %ld0
%fadd1 = fadd float %ld1, %other
-
- store float %fadd0, ptr %ptr2_0, align 4
- store float %fadd1, ptr %ptr2_1, align 4
ret void
}
; The store user bundle forms but legality packs it because the stores are
; not consecutive (there is a gap in the destination).
-define void @user_stores_not_consecutive(ptr %ptr, ptr %ptr2) {
+define void @user_stores_not_consecutive(ptr %ptr) {
; CHECK-LABEL: define void @user_stores_not_consecutive(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META12:![0-9]+]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META12]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META12]]
-; CHECK-NEXT: store float [[UNPACK]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[UNPACK1]], ptr [[PTR2_2]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
%ld0 = load float, ptr %ptr0, align 4
%ld1 = load float, ptr %ptr1, align 4
-
- store float %ld0, ptr %ptr2_0, align 4
- store float %ld1, ptr %ptr2_2, align 4
ret void
}
@@ -453,8 +349,6 @@ define void @load_fadd_store_3wide(ptr %ptr, ptr %ptr2) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META13:![0-9]+]]
; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[VECL]], !sandboxvec [[META13]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META13]]
; CHECK-NEXT: ret void
;
%g0 = getelementptr float, ptr %ptr, i32 0
@@ -467,13 +361,6 @@ define void @load_fadd_store_3wide(ptr %ptr, ptr %ptr2) {
%fadd0 = fadd float %ld0, %ld0
%fadd1 = fadd float %ld1, %ld1
%fadd2 = fadd float %ld2, %ld2
-
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
- store float %fadd0, ptr %ptr2_0
- store float %fadd1, ptr %ptr2_1
- store float %fadd2, ptr %ptr2_2
ret void
}
@@ -481,9 +368,9 @@ define void @load_fadd_store_3wide(ptr %ptr, ptr %ptr2) {
; fadd users (one adding %a, one adding %b). getNextUserBundles() must track
; claimed users across all bundles it forms so the two 3-lane bundles pick
; disjoint instructions, producing two fadd <3 x float>.
-define void @user_already_vectorized_3wide(ptr %ptr, ptr %ptr2, float %a, float %b) {
+define void @user_already_vectorized_3wide(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized_3wide(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[PACK3:%.*]] = insertelement <3 x float> poison, float [[A]], i32 0, !sandboxvec [[META14:![0-9]+]]
; CHECK-NEXT: [[PACK4:%.*]] = insertelement <3 x float> [[PACK3]], float [[A]], i32 1, !sandboxvec [[META14]]
@@ -494,10 +381,6 @@ define void @user_already_vectorized_3wide(ptr %ptr, ptr %ptr2, float %a, float
; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META14]]
; CHECK-NEXT: [[VEC6:%.*]] = fadd <3 x float> [[VECL]], [[PACK5]], !sandboxvec [[META14]]
; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK2]], !sandboxvec [[META14]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_3:%.*]] = getelementptr float, ptr [[PTR2]], i32 3
-; CHECK-NEXT: store <3 x float> [[VEC6]], ptr [[PTR2_0]], align 4, !sandboxvec [[META14]]
-; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR2_3]], align 4, !sandboxvec [[META14]]
; CHECK-NEXT: ret void
;
%g0 = getelementptr float, ptr %ptr, i32 0
@@ -513,28 +396,15 @@ define void @user_already_vectorized_3wide(ptr %ptr, ptr %ptr2, float %a, float
%fadd1b = fadd float %ld1, %b
%fadd2a = fadd float %ld2, %a
%fadd2b = fadd float %ld2, %b
-
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
- %ptr2_3 = getelementptr float, ptr %ptr2, i32 3
- %ptr2_4 = getelementptr float, ptr %ptr2, i32 4
- %ptr2_5 = getelementptr float, ptr %ptr2, i32 5
- store float %fadd0a, ptr %ptr2_0
- store float %fadd1a, ptr %ptr2_1
- store float %fadd2a, ptr %ptr2_2
- store float %fadd0b, ptr %ptr2_3
- store float %fadd1b, ptr %ptr2_4
- store float %fadd2b, ptr %ptr2_5
ret void
}
; 3-wide rejection: lanes 0 and 1 feed fadd but lane 2 feeds fmul, so the user
; bundle can't be formed. The loads still widen to <3 x float> and the
; arithmetic stays scalar, fed by unpacks.
-define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
+define void @user_opcode_mismatch_3wide(ptr %ptr) {
; CHECK-LABEL: define void @user_opcode_mismatch_3wide(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META15:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META15]]
@@ -543,12 +413,6 @@ define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT: [[FMUL2:%.*]] = fmul float [[UNPACK2]], [[UNPACK2]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
-; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
-; CHECK-NEXT: store float [[FMUL2]], ptr [[PTR2_2]], align 4
; CHECK-NEXT: ret void
;
%g0 = getelementptr float, ptr %ptr, i32 0
@@ -561,13 +425,6 @@ define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
%fadd0 = fadd float %ld0, %ld0
%fadd1 = fadd float %ld1, %ld1
%fmul2 = fmul float %ld2, %ld2
-
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
- store float %fadd0, ptr %ptr2_0
- store float %fadd1, ptr %ptr2_1
- store float %fmul2, ptr %ptr2_2
ret void
}
@@ -576,9 +433,9 @@ define void @user_opcode_mismatch_3wide(ptr %ptr, ptr %ptr2) {
; failed {fadd0a,fadd1a,?} attempt tentatively claims fadd0a/fadd1a and must roll
; them back so {fadd0b,fadd1b,fadd2b} can still form. Loads widen; the + %b
; chain vectorizes while the + %a pair stays scalar.
-define void @claimed_rollback_partial_bundle(ptr %ptr, ptr %ptr2, float %a, float %b) {
+define void @claimed_rollback_partial_bundle(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @claimed_rollback_partial_bundle(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META16:![0-9]+]]
; CHECK-NEXT: [[PACK2:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META16]]
@@ -589,12 +446,6 @@ define void @claimed_rollback_partial_bundle(ptr %ptr, ptr %ptr2, float %a, floa
; CHECK-NEXT: [[FADD0A:%.*]] = fadd float [[UNPACK]], [[A]]
; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK3]], !sandboxvec [[META16]]
; CHECK-NEXT: [[FADD1A:%.*]] = fadd float [[UNPACK1]], [[A]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
-; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
-; CHECK-NEXT: store float [[FADD0A]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FADD1A]], ptr [[PTR2_1]], align 4
-; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR2_2]], align 4, !sandboxvec [[META16]]
; CHECK-NEXT: ret void
;
%g0 = getelementptr float, ptr %ptr, i32 0
@@ -609,17 +460,6 @@ define void @claimed_rollback_partial_bundle(ptr %ptr, ptr %ptr2, float %a, floa
%fadd1a = fadd float %ld1, %a
%fadd1b = fadd float %ld1, %b
%fadd2b = fadd float %ld2, %b
-
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
- %ptr2_3 = getelementptr float, ptr %ptr2, i32 3
- %ptr2_4 = getelementptr float, ptr %ptr2, i32 4
- store float %fadd0a, ptr %ptr2_0
- store float %fadd1a, ptr %ptr2_1
- store float %fadd0b, ptr %ptr2_2
- store float %fadd1b, ptr %ptr2_3
- store float %fadd2b, ptr %ptr2_4
ret void
}
@@ -628,9 +468,9 @@ define void @claimed_rollback_partial_bundle(ptr %ptr, ptr %ptr2, float %a, floa
; lanes consecutively, so it must NOT skip the gap at lane 1 and pair lane 0's
; user with lane 2's user (which would require a shuffle). The bundle is
; rejected: loads widen to <3 x float> and all three ops stay scalar.
-define void @user_middle_lane_mismatch_3wide(ptr %ptr, ptr %ptr2) {
+define void @user_middle_lane_mismatch_3wide(ptr %ptr) {
; CHECK-LABEL: define void @user_middle_lane_mismatch_3wide(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META17:![0-9]+]]
; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META17]]
@@ -639,31 +479,18 @@ define void @user_middle_lane_mismatch_3wide(ptr %ptr, ptr %ptr2) {
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT: [[FADD2:%.*]] = fadd float [[UNPACK2]], [[UNPACK2]]
-; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
-; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
-; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
-; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
-; CHECK-NEXT: store float [[FMUL1]], ptr [[PTR2_1]], align 4
-; CHECK-NEXT: store float [[FADD2]], ptr [[PTR2_2]], align 4
; CHECK-NEXT: ret void
;
- %g0 = getelementptr float, ptr %ptr, i32 0
- %g1 = getelementptr float, ptr %ptr, i32 1
- %g2 = getelementptr float, ptr %ptr, i32 2
- %ld0 = load float, ptr %g0
- %ld1 = load float, ptr %g1
- %ld2 = load float, ptr %g2
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2 = getelementptr float, ptr %ptr, i32 2
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+ %ld2 = load float, ptr %ptr2
%fadd0 = fadd float %ld0, %ld0
%fmul1 = fmul float %ld1, %ld1
%fadd2 = fadd float %ld2, %ld2
-
- %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
- %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
- %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
- store float %fadd0, ptr %ptr2_0
- store float %fmul1, ptr %ptr2_1
- store float %fadd2, ptr %ptr2_2
ret void
}
>From d5047dfb85b427fc050be9bba6ef0ad6a44e2ba5 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Wed, 29 Jul 2026 22:32:25 -0400
Subject: [PATCH 09/11] Refactor getNextUserBundles
---
.../Vectorize/SandboxVectorizer/VecUtils.h | 2 +
.../Vectorize/SandboxVectorizer/VecUtils.cpp | 104 +++----
.../SandboxVectorizer/topdown_vec.ll | 276 ++++++++++++------
.../SandboxVectorizer/VecUtilsTest.cpp | 31 +-
4 files changed, 242 insertions(+), 171 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index 2504174591993..f2e571fc9f649 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -231,6 +231,8 @@ class VecUtils {
/// For each user of lane 0 in \p Bndl, try to form a bundle of matching
/// users for all lanes. Returns all complete user bundles found.
+ /// \p Claimed contains instructions that have already been claimed by a
+ /// bundle.
LLVM_ABI static SmallVector<BundleTy>
getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
SmallPtrSet<Instruction *, 4> &Claimed);
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
index 29c8edbcc2427..333ad2f37f0e5 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
@@ -15,6 +15,44 @@
namespace llvm::sandboxir {
+static SmallVector<unsigned, 2> getOperandIndicesInUser(User *U, Value *Op) {
+ SmallVector<unsigned, 2> OpIdxVec;
+ for (unsigned Idx : seq<unsigned>(U->getNumOperands()))
+ if (U->getOperand(Idx) == Op)
+ OpIdxVec.push_back(Idx);
+ return OpIdxVec;
+}
+
+static std::optional<BundleTy> getMatchingBundle(ArrayRef<Value *> Bndl, const InstrMaps &IMaps, Value *Seed, Instruction *SeedUserInst, SmallPtrSet<Instruction *, 4> &Claimed) {
+ SmallVector<unsigned, 2> OpIdxVec0 = getOperandIndicesInUser(SeedUserInst, Seed);
+ assert(!OpIdxVec0.empty() && "U0 does not use Seed!");
+ BundleTy NextUserBndl;
+ NextUserBndl.push_back(SeedUserInst);
+ Claimed.insert(SeedUserInst);
+ for (Value *V : drop_begin(Bndl)) {
+ Instruction *Match = nullptr;
+ for (User *U : V->users()) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || IMaps.isVectorized(UI) || Claimed.contains(UI) ||
+ UI->getOpcode() != SeedUserInst->getOpcode() ||
+ UI->getType() != SeedUserInst->getType() ||
+ UI->getParent() != SeedUserInst->getParent() ||
+ getOperandIndicesInUser(UI, V) != OpIdxVec0)
+ continue;
+
+ Match = UI;
+ break;
+ }
+ if (!Match)
+ return std::nullopt;
+ NextUserBndl.push_back(Match);
+ }
+
+ for (auto *I : NextUserBndl)
+ Claimed.insert(cast<Instruction>(I));
+ return NextUserBndl;
+}
+
SmallVector<BundleTy>
VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
SmallPtrSet<Instruction *, 4> &Claimed) {
@@ -22,16 +60,6 @@ VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
if (Bndl.empty())
return Bundles;
- // Collect the operand indices at which \p U uses \p V. Operands are scanned
- // in ascending order, so the result is sorted.
- auto GetOpIdxVec = [](Value *V, User *U) -> SmallVector<unsigned, 2> {
- SmallVector<unsigned, 2> OpIdxVec;
- for (unsigned Idx : seq<unsigned>(U->getNumOperands()))
- if (U->getOperand(Idx) == V)
- OpIdxVec.push_back(Idx);
- return OpIdxVec;
- };
-
Value *V0 = Bndl[0];
DenseSet<User *> SeenUsers;
// For each user U0 of lane 0, try to form a bundle of matching users across
@@ -42,59 +70,9 @@ VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
auto *UI0 = dyn_cast<Instruction>(U0);
if (!UI0 || IMaps.isVectorized(UI0) || Claimed.contains(UI0))
continue;
-
- // The operand indices at which lane 0's user U0 uses lane 0's value V0.
- // Every other lane's user must use its lane value at the exact same operand
- // indices; otherwise the widened user's operands can't be grouped
- // consistently (each vector operand lane must come from the same position).
- SmallVector<unsigned, 2> OpIdxVec0 = GetOpIdxVec(V0, UI0);
- assert(!OpIdxVec0.empty() && "U0 does not use V0!");
-
- // Find a distinct matching user for each of the remaining lanes.
- BundleTy NextUserBndl;
- NextUserBndl.push_back(UI0);
- // Tentatively claim UI0; roll back if a full bundle can't be formed.
- SmallVector<Instruction *, 4> NewlyClaimed;
- Claimed.insert(UI0);
- NewlyClaimed.push_back(UI0);
- for (Value *V : drop_begin(Bndl)) {
- Instruction *Match = nullptr;
- for (User *U : V->users()) {
- auto *UI = dyn_cast<Instruction>(U);
- if (!UI || IMaps.isVectorized(UI) || Claimed.contains(UI))
- continue;
- if (UI->getOpcode() != UI0->getOpcode() ||
- UI->getType() != UI0->getType())
- continue;
- if (UI->getParent() != UI0->getParent())
- continue;
-
- // Require the same operand-usage pattern as lane 0 (same indices, in
- // order). This rejects both operand-index mismatches and cases where V
- // is used a different number of times than V0 is in U0.
- if (GetOpIdxVec(V, UI) != OpIdxVec0)
- continue;
-
- Match = UI;
- break;
- }
- if (!Match) {
- NextUserBndl.clear();
- break;
- }
- Claimed.insert(Match);
- NewlyClaimed.push_back(Match);
- NextUserBndl.push_back(Match);
- }
-
- if (NextUserBndl.size() == Bndl.size()) {
- Bundles.emplace_back(std::move(NextUserBndl));
- } else {
- // Failed to form a full bundle; release the instructions we tentatively
- // claimed so they remain available for other lane-0 users.
- for (Instruction *I : NewlyClaimed)
- Claimed.erase(I);
- }
+ std::optional<BundleTy> NextUserBndl = getMatchingBundle(Bndl, IMaps, V0, UI0, Claimed);
+ if (NextUserBndl)
+ Bundles.emplace_back(std::move(*NextUserBndl));
}
return Bundles;
}
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
index 81b5eec014ee6..7355753d8ae8c 100644
--- a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -1,8 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-collect-seeds=loads -sbvec-passes="seed-collection<tr-save,bottom-up-vec(top-down),tr-accept>" %s -S | FileCheck %s
-; Tests: successful bundle match (baseline) and !IMaps->isVectorized(UI0) on
-; the outer loop's second use edge (%ld0 used twice by the same fadd).
define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
; CHECK-LABEL: define void @load_fadd_store(
; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
@@ -65,9 +63,6 @@ define float @load_fadd_external_use(ptr %ptr) {
ret float %fadd0
}
-; Both lanes feed the *same* user instruction (once per operand). The user
-; bundle must not be formed out of duplicate instructions, so the fadd stays
-; scalar while the loads still widen.
define float @single_user_no_duplicate(ptr %ptr) {
; CHECK-LABEL: define float @single_user_no_duplicate(
; CHECK-SAME: ptr [[PTR:%.*]]) {
@@ -87,8 +82,6 @@ define float @single_user_no_duplicate(ptr %ptr) {
ret float %fadd
}
-; The candidate users live in different blocks from each other, so they must
-; not be bundled together.
define void @users_in_different_blocks(ptr %ptr, i1 %c) {
; CHECK-LABEL: define void @users_in_different_blocks(
; CHECK-SAME: ptr [[PTR:%.*]], i1 [[C:%.*]]) {
@@ -121,7 +114,6 @@ if.else:
ret void
}
-; Lane 0 feeds fadd, lane 1 feeds fmul — opcode mismatch rejects the bundle.
define void @user_opcode_mismatch(ptr %ptr) {
; CHECK-LABEL: define void @user_opcode_mismatch(
; CHECK-SAME: ptr [[PTR:%.*]]) {
@@ -144,7 +136,6 @@ define void @user_opcode_mismatch(ptr %ptr) {
ret void
}
-; Lane 0's user is fadd float, lane 1's user is fadd double — type mismatch.
define void @user_type_mismatch(ptr %ptr) {
; CHECK-LABEL: define void @user_type_mismatch(
; CHECK-SAME: ptr [[PTR:%.*]]) {
@@ -169,8 +160,6 @@ define void @user_type_mismatch(ptr %ptr) {
ret void
}
-; Lane 0 uses ld0 at operand 0; lane 1 uses ld1 at operand 1 (fsub is not
-; commutative). Operand-index mismatch rejects the bundle.
define void @user_operand_index_mismatch(ptr %ptr, float %x) {
; CHECK-LABEL: define void @user_operand_index_mismatch(
; CHECK-SAME: ptr [[PTR:%.*]], float [[X:%.*]]) {
@@ -193,22 +182,44 @@ define void @user_operand_index_mismatch(ptr %ptr, float %x) {
ret void
}
-; ld0/ld1 each have two fadd users: one adding %a and one adding %b. The two
-; user bundles must not fight over the same instruction: BottomUpVec passes a
-; Claimed set into getNextUserBundles(), so once {fadd0b,fadd1b} is claimed,
-; the second bundle picks {fadd0,fadd1} rather than reusing fadd1b.
-; Both bundles widen, producing two fadd <2 x float>.
+define void @user_operand_index_mismatch_3wide(ptr %ptr, float %x) {
+; CHECK-LABEL: define void @user_operand_index_mismatch_3wide(
+; CHECK-SAME: ptr [[PTR:%.*]], float [[X:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META8:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META8]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META8]]
+; CHECK-NEXT: [[FSUB0:%.*]] = fsub float [[UNPACK]], [[X]]
+; CHECK-NEXT: [[FSUB1:%.*]] = fsub float [[X]], [[UNPACK1]]
+; CHECK-NEXT: [[FSUB2:%.*]] = fsub float [[UNPACK2]], 0.000000e+00
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2 = getelementptr float, ptr %ptr, i32 2
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+ %ld2 = load float, ptr %ptr2, align 4
+
+ %fsub0 = fsub float %ld0, %x
+ %fsub1 = fsub float %x, %ld1
+ %fsub2 = fsub float %ld2, 0.0
+ ret void
+}
+
define void @user_already_vectorized(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META8:![0-9]+]]
-; CHECK-NEXT: [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META8]]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META8]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META8]]
-; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META8]]
-; CHECK-NEXT: [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META8]]
-; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META8]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META9:![0-9]+]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META9]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META9]]
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META9]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META9]]
+; CHECK-NEXT: [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META9]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META9]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
@@ -221,20 +232,113 @@ define void @user_already_vectorized(ptr %ptr, float %a, float %b) {
%fadd1 = fadd float %ld1, %a
%fadd0b = fadd float %ld0, %b
%fadd1b = fadd float %ld1, %b
+ ret void
+}
+
+define void @multiple_users_of_loads(ptr %ptr, float %a, float %b, float %c) {
+; CHECK-LABEL: define void @multiple_users_of_loads(
+; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PACK5:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META10:![0-9]+]]
+; CHECK-NEXT: [[PACK6:%.*]] = insertelement <2 x float> [[PACK5]], float [[A]], i32 1, !sandboxvec [[META10]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META10]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[B]], i32 1, !sandboxvec [[META10]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[C]], i32 0, !sandboxvec [[META10]]
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[C]], i32 1, !sandboxvec [[META10]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META10]]
+; CHECK-NEXT: [[VEC7:%.*]] = fadd <2 x float> [[VECL]], [[PACK6]], !sandboxvec [[META10]]
+; CHECK-NEXT: [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META10]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META10]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0a = fadd float %ld0, %a
+ %fadd0b = fadd float %ld0, %b
+ %fadd0c = fadd float %ld0, %c
+
+ %fadd1a = fadd float %ld1, %a
+ %fadd1b = fadd float %ld1, %b
+ %fadd1c = fadd float %ld1, %c
+ ret void
+}
+
+define void @multiple_users_of_loads_diff_opcode(ptr %ptr, float %a, float %b) {
+; CHECK-LABEL: define void @multiple_users_of_loads_diff_opcode(
+; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META11:![0-9]+]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META11]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META11]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META11]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META11]]
+; CHECK-NEXT: [[FADD0A:%.*]] = fadd float [[UNPACK]], [[A]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK2]], !sandboxvec [[META11]]
+; CHECK-NEXT: [[FSUB1A:%.*]] = fsub float [[UNPACK1]], [[A]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0a = fadd float %ld0, %a
+ %fadd0b = fadd float %ld0, %b
+
+ %fsub1a = fsub float %ld1, %a
+ %fadd1b = fadd float %ld1, %b
+ ret void
+}
+
+define void @users_in_arbitrary_order(ptr %ptr, float %a, float %b, float %c) {
+; CHECK-LABEL: define void @users_in_arbitrary_order(
+; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PACK8:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META12:![0-9]+]]
+; CHECK-NEXT: [[PACK9:%.*]] = insertelement <2 x float> [[PACK8]], float [[A]], i32 1, !sandboxvec [[META12]]
+; CHECK-NEXT: [[PACK5:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META12]]
+; CHECK-NEXT: [[PACK6:%.*]] = insertelement <2 x float> [[PACK5]], float [[B]], i32 1, !sandboxvec [[META12]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META12]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META12]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[C]], i32 0, !sandboxvec [[META12]]
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[C]], i32 1, !sandboxvec [[META12]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META12]]
+; CHECK-NEXT: [[VEC10:%.*]] = fadd <2 x float> [[VECL]], [[PACK9]], !sandboxvec [[META12]]
+; CHECK-NEXT: [[VEC7:%.*]] = fadd <2 x float> [[VECL]], [[PACK6]], !sandboxvec [[META12]]
+; CHECK-NEXT: [[VEC4:%.*]] = fsub <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META12]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META12]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0a = fadd float %ld0, %a
+ %fadd1a = fadd float %ld1, %a
+ %fadd0b = fadd float %ld0, %b
+ %fadd1b = fadd float %ld1, %b
+ %fsub0a = fsub float %ld0, %a
+ %fadd0c = fadd float %ld0, %c
+ %fadd1c = fadd float %ld1, %c
+ %fsub1a = fsub float %ld1, %a
ret void
}
-; The fadd user bundle passes the getNextUserBundles() checks (same opcode,
-; type, BB, operand index) but legality returns Pack due to different
-; fast-math flags. The recursion must stop there: loads widen, fadds stay
-; scalar and are fed by unpacks.
+
define void @user_diff_fast_math_flags(ptr %ptr) {
; CHECK-LABEL: define void @user_diff_fast_math_flags(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META9:![0-9]+]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META9]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META9]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META13:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META13]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META13]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT: ret void
@@ -250,15 +354,13 @@ define void @user_diff_fast_math_flags(ptr %ptr) {
ret void
}
-; Same as above but the user bundle packs due to different wrap flags
-; (add nsw vs add).
define void @user_diff_wrap_flags(ptr %ptr) {
; CHECK-LABEL: define void @user_diff_wrap_flags(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 4, !sandboxvec [[META10:![0-9]+]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i32> [[VECL]], i32 0, !sandboxvec [[META10]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i32> [[VECL]], i32 1, !sandboxvec [[META10]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 4, !sandboxvec [[META14:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i32> [[VECL]], i32 0, !sandboxvec [[META14]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i32> [[VECL]], i32 1, !sandboxvec [[META14]]
; CHECK-NEXT: [[ADD0:%.*]] = add nsw i32 [[UNPACK]], 1
; CHECK-NEXT: [[ADD1:%.*]] = add i32 [[UNPACK1]], 1
; CHECK-NEXT: ret void
@@ -274,17 +376,13 @@ define void @user_diff_wrap_flags(ptr %ptr) {
ret void
}
-; Lane 0's value is used twice in %fadd0 (operands {0,1}) but lane 1's matching
-; user only uses %ld1 once (at operand 0). The operand-usage patterns differ, so
-; getNextUserBundles() rejects the bundle: the loads widen while the fadds stay
-; scalar and are fed by unpacks.
define void @user_duplicate_operand_other(ptr %ptr, float %other) {
; CHECK-LABEL: define void @user_duplicate_operand_other(
; CHECK-SAME: ptr [[PTR:%.*]], float [[OTHER:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META11:![0-9]+]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META11]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META11]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META15:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META15]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META15]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[OTHER]]
; CHECK-NEXT: ret void
@@ -300,13 +398,11 @@ define void @user_duplicate_operand_other(ptr %ptr, float %other) {
ret void
}
-; The store user bundle forms but legality packs it because the stores are
-; not consecutive (there is a gap in the destination).
define void @user_stores_not_consecutive(ptr %ptr) {
; CHECK-LABEL: define void @user_stores_not_consecutive(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META12:![0-9]+]]
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META16:![0-9]+]]
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
@@ -342,21 +438,20 @@ define void @user_stores_not_consecutive(ptr %ptr) {
; ret void
;}
-; 3-wide baseline: three lanes widen through load -> fadd -> store.
-define void @load_fadd_store_3wide(ptr %ptr, ptr %ptr2) {
+define void @load_fadd_store_3wide(ptr %ptr) {
; CHECK-LABEL: define void @load_fadd_store_3wide(
-; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META13:![0-9]+]]
-; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[VECL]], !sandboxvec [[META13]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META17:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[VECL]], !sandboxvec [[META17]]
; CHECK-NEXT: ret void
;
- %g0 = getelementptr float, ptr %ptr, i32 0
- %g1 = getelementptr float, ptr %ptr, i32 1
- %g2 = getelementptr float, ptr %ptr, i32 2
- %ld0 = load float, ptr %g0
- %ld1 = load float, ptr %g1
- %ld2 = load float, ptr %g2
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2 = getelementptr float, ptr %ptr, i32 2
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+ %ld2 = load float, ptr %ptr2
%fadd0 = fadd float %ld0, %ld0
%fadd1 = fadd float %ld1, %ld1
@@ -364,23 +459,19 @@ define void @load_fadd_store_3wide(ptr %ptr, ptr %ptr2) {
ret void
}
-; 3-wide version of @user_already_vectorized: each of the three loads feeds two
-; fadd users (one adding %a, one adding %b). getNextUserBundles() must track
-; claimed users across all bundles it forms so the two 3-lane bundles pick
-; disjoint instructions, producing two fadd <3 x float>.
define void @user_already_vectorized_3wide(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized_3wide(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PACK3:%.*]] = insertelement <3 x float> poison, float [[A]], i32 0, !sandboxvec [[META14:![0-9]+]]
-; CHECK-NEXT: [[PACK4:%.*]] = insertelement <3 x float> [[PACK3]], float [[A]], i32 1, !sandboxvec [[META14]]
-; CHECK-NEXT: [[PACK5:%.*]] = insertelement <3 x float> [[PACK4]], float [[A]], i32 2, !sandboxvec [[META14]]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META14]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META14]]
-; CHECK-NEXT: [[PACK2:%.*]] = insertelement <3 x float> [[PACK1]], float [[B]], i32 2, !sandboxvec [[META14]]
-; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META14]]
-; CHECK-NEXT: [[VEC6:%.*]] = fadd <3 x float> [[VECL]], [[PACK5]], !sandboxvec [[META14]]
-; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK2]], !sandboxvec [[META14]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <3 x float> poison, float [[A]], i32 0, !sandboxvec [[META18:![0-9]+]]
+; CHECK-NEXT: [[PACK4:%.*]] = insertelement <3 x float> [[PACK3]], float [[A]], i32 1, !sandboxvec [[META18]]
+; CHECK-NEXT: [[PACK5:%.*]] = insertelement <3 x float> [[PACK4]], float [[A]], i32 2, !sandboxvec [[META18]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META18]]
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META18]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <3 x float> [[PACK1]], float [[B]], i32 2, !sandboxvec [[META18]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META18]]
+; CHECK-NEXT: [[VEC6:%.*]] = fadd <3 x float> [[VECL]], [[PACK5]], !sandboxvec [[META18]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK2]], !sandboxvec [[META18]]
; CHECK-NEXT: ret void
;
%g0 = getelementptr float, ptr %ptr, i32 0
@@ -399,17 +490,14 @@ define void @user_already_vectorized_3wide(ptr %ptr, float %a, float %b) {
ret void
}
-; 3-wide rejection: lanes 0 and 1 feed fadd but lane 2 feeds fmul, so the user
-; bundle can't be formed. The loads still widen to <3 x float> and the
-; arithmetic stays scalar, fed by unpacks.
define void @user_opcode_mismatch_3wide(ptr %ptr) {
; CHECK-LABEL: define void @user_opcode_mismatch_3wide(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META15:![0-9]+]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META15]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META15]]
-; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META15]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META19:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META19]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META19]]
+; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META19]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT: [[FMUL2:%.*]] = fmul float [[UNPACK2]], [[UNPACK2]]
@@ -428,23 +516,18 @@ define void @user_opcode_mismatch_3wide(ptr %ptr) {
ret void
}
-; 3-wide Claimed rollback: each load has + %a and + %b fadd users, but ld2 has
-; no + %a user. BottomUpVec passes a Claimed set into getNextUserBundles(); a
-; failed {fadd0a,fadd1a,?} attempt tentatively claims fadd0a/fadd1a and must roll
-; them back so {fadd0b,fadd1b,fadd2b} can still form. Loads widen; the + %b
-; chain vectorizes while the + %a pair stays scalar.
define void @claimed_rollback_partial_bundle(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @claimed_rollback_partial_bundle(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META16:![0-9]+]]
-; CHECK-NEXT: [[PACK2:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META16]]
-; CHECK-NEXT: [[PACK3:%.*]] = insertelement <3 x float> [[PACK2]], float [[B]], i32 2, !sandboxvec [[META16]]
-; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META16]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META16]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META16]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META20:![0-9]+]]
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META20]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <3 x float> [[PACK2]], float [[B]], i32 2, !sandboxvec [[META20]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META20]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META20]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META20]]
; CHECK-NEXT: [[FADD0A:%.*]] = fadd float [[UNPACK]], [[A]]
-; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK3]], !sandboxvec [[META16]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK3]], !sandboxvec [[META20]]
; CHECK-NEXT: [[FADD1A:%.*]] = fadd float [[UNPACK1]], [[A]]
; CHECK-NEXT: ret void
;
@@ -463,19 +546,14 @@ define void @claimed_rollback_partial_bundle(ptr %ptr, float %a, float %b) {
ret void
}
-; 3-wide non-consecutive rejection: lane 0 (fadd) and lane 2 (fadd) have
-; matching users but the middle lane 1 uses fmul. getNextUserBundles() matches
-; lanes consecutively, so it must NOT skip the gap at lane 1 and pair lane 0's
-; user with lane 2's user (which would require a shuffle). The bundle is
-; rejected: loads widen to <3 x float> and all three ops stay scalar.
define void @user_middle_lane_mismatch_3wide(ptr %ptr) {
; CHECK-LABEL: define void @user_middle_lane_mismatch_3wide(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
-; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META17:![0-9]+]]
-; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META17]]
-; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META17]]
-; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META17]]
+; CHECK-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META21:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META21]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META21]]
+; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META21]]
; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT: [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT: [[FADD2:%.*]] = fadd float [[UNPACK2]], [[UNPACK2]]
@@ -513,4 +591,8 @@ define void @user_middle_lane_mismatch_3wide(ptr %ptr) {
; CHECK: [[META15]] = distinct !{!"sandboxregion"}
; CHECK: [[META16]] = distinct !{!"sandboxregion"}
; CHECK: [[META17]] = distinct !{!"sandboxregion"}
+; CHECK: [[META18]] = distinct !{!"sandboxregion"}
+; CHECK: [[META19]] = distinct !{!"sandboxregion"}
+; CHECK: [[META20]] = distinct !{!"sandboxregion"}
+; CHECK: [[META21]] = distinct !{!"sandboxregion"}
;.
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
index 84b8812ca0db3..da888d43ef691 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -925,12 +925,13 @@ define void @vectorized_seed_user(ptr %p) {
auto *Ld1 = &*It++;
auto *Add0 = &*It++;
auto *Add1 = &*It++;
-
+
ASSERT_EQ(Add0->getOperand(0), Ld0);
ASSERT_EQ(Add1->getOperand(0), Ld1);
-
+
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
auto NextUserBundles =
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps);
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed);
ASSERT_EQ(NextUserBundles.size(), 1u);
ASSERT_EQ(NextUserBundles[0].size(), 2u);
EXPECT_EQ(NextUserBundles[0][0], Add0);
@@ -944,8 +945,9 @@ define void @vectorized_seed_user(ptr %p) {
std::advance(It, 2);
auto *Ld0 = &*It++;
auto *Ld1 = &*It++;
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
});
withFunction(
@@ -954,8 +956,9 @@ define void @vectorized_seed_user(ptr %p) {
auto It = BB.begin();
auto *F0 = &*It++;
auto *D1 = &*It++;
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({F0, D1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({F0, D1}, IMaps, Claimed).empty());
});
withFunction("block_mismatch", [](sandboxir::Function &F,
@@ -965,8 +968,9 @@ define void @vectorized_seed_user(ptr %p) {
std::advance(It, 2);
auto *Ld0 = &*It++;
auto *Ld1 = &*It++;
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
});
withFunction("operand_mismatch", [](sandboxir::Function &F,
@@ -976,8 +980,9 @@ define void @vectorized_seed_user(ptr %p) {
std::advance(It, 2);
auto *Ld0 = &*It++;
auto *Ld1 = &*It++;
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
});
withFunction("duplicate_operand_mismatch", [](sandboxir::Function &F,
@@ -989,8 +994,9 @@ define void @vectorized_seed_user(ptr %p) {
auto *Ld1 = &*It++;
// Lane 0's user uses Ld0 at operands {0, 1} but lane 1's user
// uses Ld1 only at operand {0}, so no bundle should form.
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
});
withFunction("missing_lane_user", [](sandboxir::Function &F,
@@ -1000,8 +1006,9 @@ define void @vectorized_seed_user(ptr %p) {
std::advance(It, 2);
auto *Ld0 = &*It++;
auto *Ld1 = &*It++;
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
});
withFunction("vectorized_user", [](sandboxir::Function &F,
@@ -1016,8 +1023,9 @@ define void @vectorized_seed_user(ptr %p) {
(void)Add0;
sandboxir::Action A(nullptr, {Add1}, {}, 0);
IMaps.registerVector({Add1}, &A);
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
});
withFunction("vectorized_seed_user", [](sandboxir::Function &F,
@@ -1030,7 +1038,8 @@ define void @vectorized_seed_user(ptr %p) {
auto *Add0 = &*It++;
sandboxir::Action A(nullptr, {Add0}, {}, 0);
IMaps.registerVector({Add0}, &A);
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps).empty());
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
});
}
>From 96b251430bb8a78647714a60fcb2586c5d353259 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Thu, 30 Jul 2026 00:19:18 -0400
Subject: [PATCH 10/11] Add a cl::opt to limit the number of users visited
---
.../Vectorize/SandboxVectorizer/VecUtils.cpp | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
index 333ad2f37f0e5..49b1fb7db285f 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
@@ -11,10 +11,17 @@
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
namespace llvm::sandboxir {
+static cl::opt<unsigned> MaxUsersToConsider(
+ "sbvec-max-users-to-consider", cl::init(16), cl::Hidden,
+ cl::desc("Limit the number of a seed's users that getNextUserBundles() "
+ "will examine as candidates for a matching bundle, to cap "
+ "compilation time."));
+
static SmallVector<unsigned, 2> getOperandIndicesInUser(User *U, Value *Op) {
SmallVector<unsigned, 2> OpIdxVec;
for (unsigned Idx : seq<unsigned>(U->getNumOperands()))
@@ -63,8 +70,12 @@ VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
Value *V0 = Bndl[0];
DenseSet<User *> SeenUsers;
// For each user U0 of lane 0, try to form a bundle of matching users across
- // all lanes.
+ // all lanes. Cap the number of users considered to bound compilation time,
+ // since each one may trigger an O(Bndl.size()) search across the other
+ // lanes' users.
for (User *U0 : V0->users()) {
+ if (SeenUsers.size() >= MaxUsersToConsider)
+ break;
if (!SeenUsers.insert(U0).second)
continue;
auto *UI0 = dyn_cast<Instruction>(U0);
>From 7862d524a4712ec3f80391004c4d77a8d0b74a71 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Thu, 30 Jul 2026 00:30:11 -0400
Subject: [PATCH 11/11] Refactor VecUtilsTest for readability
---
.../Vectorize/SandboxVectorizer/VecUtils.cpp | 11 +-
.../SandboxVectorizer/VecUtilsTest.cpp | 299 ++++++++++--------
2 files changed, 172 insertions(+), 138 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
index 49b1fb7db285f..bf8f66d00e7eb 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/VecUtils.cpp
@@ -30,8 +30,12 @@ static SmallVector<unsigned, 2> getOperandIndicesInUser(User *U, Value *Op) {
return OpIdxVec;
}
-static std::optional<BundleTy> getMatchingBundle(ArrayRef<Value *> Bndl, const InstrMaps &IMaps, Value *Seed, Instruction *SeedUserInst, SmallPtrSet<Instruction *, 4> &Claimed) {
- SmallVector<unsigned, 2> OpIdxVec0 = getOperandIndicesInUser(SeedUserInst, Seed);
+static std::optional<BundleTy>
+getMatchingBundle(ArrayRef<Value *> Bndl, const InstrMaps &IMaps, Value *Seed,
+ Instruction *SeedUserInst,
+ SmallPtrSet<Instruction *, 4> &Claimed) {
+ SmallVector<unsigned, 2> OpIdxVec0 =
+ getOperandIndicesInUser(SeedUserInst, Seed);
assert(!OpIdxVec0.empty() && "U0 does not use Seed!");
BundleTy NextUserBndl;
NextUserBndl.push_back(SeedUserInst);
@@ -81,7 +85,8 @@ VecUtils::getNextUserBundles(ArrayRef<Value *> Bndl, const InstrMaps &IMaps,
auto *UI0 = dyn_cast<Instruction>(U0);
if (!UI0 || IMaps.isVectorized(UI0) || Claimed.contains(UI0))
continue;
- std::optional<BundleTy> NextUserBndl = getMatchingBundle(Bndl, IMaps, V0, UI0, Claimed);
+ std::optional<BundleTy> NextUserBndl =
+ getMatchingBundle(Bndl, IMaps, V0, UI0, Claimed);
if (NextUserBndl)
Bundles.emplace_back(std::move(*NextUserBndl));
}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
index da888d43ef691..ec9e5f200442f 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -808,7 +808,7 @@ define void @foo(i32 %s0, <4 x i32> %v0, i32 %s1, <2 x i32> %v1, <3 x i32> %v2,
EXPECT_THAT(Lanes, testing::ElementsAre(0, 1, 5, 6, 8, 11));
}
-TEST_F(VecUtilsTest, GetNextUserBundle) {
+TEST_F(VecUtilsTest, GetNextUserBundle_Match) {
parseIR(R"IR(
define void @match(ptr %p) {
entry:
@@ -820,7 +820,32 @@ define void @match(ptr %p) {
%add1 = fadd float %ld1, %ld1
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("match"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ auto *Add0 = &*It++;
+ auto *Add1 = &*It++;
+
+ ASSERT_EQ(Add0->getOperand(0), Ld0);
+ ASSERT_EQ(Add1->getOperand(0), Ld1);
+
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ auto NextUserBundles =
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed);
+ ASSERT_EQ(NextUserBundles.size(), 1u);
+ ASSERT_EQ(NextUserBundles[0].size(), 2u);
+ EXPECT_EQ(NextUserBundles[0][0], Add0);
+ EXPECT_EQ(NextUserBundles[0][1], Add1);
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_OpcodeMismatch) {
+ parseIR(R"IR(
define void @opcode_mismatch(ptr %p) {
entry:
%gep0 = getelementptr float, ptr %p, i32 0
@@ -831,7 +856,24 @@ define void @opcode_mismatch(ptr %p) {
%sub1 = fsub float %ld1, %ld1
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("opcode_mismatch"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed)
+ .empty());
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_TypeMismatch) {
+ parseIR(R"IR(
define void @type_mismatch(ptr %pf, ptr %pd) {
entry:
%f0 = load float, ptr %pf
@@ -840,7 +882,22 @@ define void @type_mismatch(ptr %pf, ptr %pd) {
%add1 = fadd double %d1, %d1
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("type_mismatch"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ auto *F0 = &*It++;
+ auto *D1 = &*It++;
+
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(sandboxir::VecUtils::getNextUserBundles({F0, D1}, IMaps, Claimed)
+ .empty());
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_BlockMismatch) {
+ parseIR(R"IR(
define void @block_mismatch(ptr %p) {
entry:
%gep0 = getelementptr float, ptr %p, i32 0
@@ -853,7 +910,24 @@ define void @block_mismatch(ptr %p) {
%add1 = fadd float %ld1, %ld1
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("block_mismatch"));
+ auto &Entry = getBasicBlockByName(*F, "entry");
+ auto It = Entry.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed)
+ .empty());
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_OperandMismatch) {
+ parseIR(R"IR(
define void @operand_mismatch(ptr %p) {
entry:
%gep0 = getelementptr float, ptr %p, i32 0
@@ -864,7 +938,24 @@ define void @operand_mismatch(ptr %p) {
%add1 = fadd float %ld0, %ld1
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("operand_mismatch"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed)
+ .empty());
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_DuplicateOperandMismatch) {
+ parseIR(R"IR(
define void @duplicate_operand_mismatch(ptr %p, float %other) {
entry:
%gep0 = getelementptr float, ptr %p, i32 0
@@ -875,7 +966,25 @@ define void @duplicate_operand_mismatch(ptr %p, float %other) {
%add1 = fadd float %ld1, %other
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("duplicate_operand_mismatch"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ // Lane 0's user uses Ld0 at operands {0, 1} but lane 1's user
+ // uses Ld1 only at operand {0}, so no bundle should form.
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed)
+ .empty());
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_MissingLaneUser) {
+ parseIR(R"IR(
define void @missing_lane_user(ptr %p) {
entry:
%gep0 = getelementptr float, ptr %p, i32 0
@@ -885,7 +994,24 @@ define void @missing_lane_user(ptr %p) {
%add0 = fadd float %ld0, %ld0
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("missing_lane_user"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed)
+ .empty());
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_VectorizedUser) {
+ parseIR(R"IR(
define void @vectorized_user(ptr %p) {
entry:
%gep0 = getelementptr float, ptr %p, i32 0
@@ -896,7 +1022,28 @@ define void @vectorized_user(ptr %p) {
%add1 = fadd float %ld1, %ld1
ret void
}
+)IR");
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("vectorized_user"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ It++; // Add0, unused: only Add1 is marked vectorized below.
+ auto *Add1 = &*It++;
+
+ sandboxir::Action A(nullptr, {Add1}, {}, 0);
+ IMaps.registerVector({Add1}, &A);
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed)
+ .empty());
+}
+TEST_F(VecUtilsTest, GetNextUserBundle_VectorizedSeedUser) {
+ parseIR(R"IR(
define void @vectorized_seed_user(ptr %p) {
entry:
%gep0 = getelementptr float, ptr %p, i32 0
@@ -908,138 +1055,20 @@ define void @vectorized_seed_user(ptr %p) {
ret void
}
)IR");
-
- auto withFunction = [this](StringRef FuncName, auto &&TestFn) {
- sandboxir::Context Ctx(C);
- sandboxir::InstrMaps IMaps;
- auto *F = Ctx.createFunction(M->getFunction(FuncName));
- TestFn(*F, IMaps);
- };
-
- withFunction("match",
- [](sandboxir::Function &F, sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- auto *Add0 = &*It++;
- auto *Add1 = &*It++;
-
- ASSERT_EQ(Add0->getOperand(0), Ld0);
- ASSERT_EQ(Add1->getOperand(0), Ld1);
-
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- auto NextUserBundles =
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed);
- ASSERT_EQ(NextUserBundles.size(), 1u);
- ASSERT_EQ(NextUserBundles[0].size(), 2u);
- EXPECT_EQ(NextUserBundles[0][0], Add0);
- EXPECT_EQ(NextUserBundles[0][1], Add1);
- });
-
- withFunction("opcode_mismatch", [](sandboxir::Function &F,
- sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
- });
-
- withFunction(
- "type_mismatch", [](sandboxir::Function &F, sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- auto *F0 = &*It++;
- auto *D1 = &*It++;
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({F0, D1}, IMaps, Claimed).empty());
- });
-
- withFunction("block_mismatch", [](sandboxir::Function &F,
- sandboxir::InstrMaps &IMaps) {
- auto &Entry = getBasicBlockByName(F, "entry");
- auto It = Entry.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
- });
-
- withFunction("operand_mismatch", [](sandboxir::Function &F,
- sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
- });
-
- withFunction("duplicate_operand_mismatch", [](sandboxir::Function &F,
- sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- // Lane 0's user uses Ld0 at operands {0, 1} but lane 1's user
- // uses Ld1 only at operand {0}, so no bundle should form.
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
- });
-
- withFunction("missing_lane_user", [](sandboxir::Function &F,
- sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
- });
-
- withFunction("vectorized_user", [](sandboxir::Function &F,
- sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- auto *Add0 = &*It++;
- auto *Add1 = &*It++;
- (void)Add0;
- sandboxir::Action A(nullptr, {Add1}, {}, 0);
- IMaps.registerVector({Add1}, &A);
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
- });
-
- withFunction("vectorized_seed_user", [](sandboxir::Function &F,
- sandboxir::InstrMaps &IMaps) {
- auto &BB = getBasicBlockByName(F, "entry");
- auto It = BB.begin();
- std::advance(It, 2);
- auto *Ld0 = &*It++;
- auto *Ld1 = &*It++;
- auto *Add0 = &*It++;
- sandboxir::Action A(nullptr, {Add0}, {}, 0);
- IMaps.registerVector({Add0}, &A);
- SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
- EXPECT_TRUE(
- sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed).empty());
- });
+ sandboxir::Context Ctx(C);
+ sandboxir::InstrMaps IMaps;
+ auto *F = Ctx.createFunction(M->getFunction("vectorized_seed_user"));
+ auto &BB = getBasicBlockByName(*F, "entry");
+ auto It = BB.begin();
+ std::advance(It, 2);
+ auto *Ld0 = &*It++;
+ auto *Ld1 = &*It++;
+ auto *Add0 = &*It++;
+
+ sandboxir::Action A(nullptr, {Add0}, {}, 0);
+ IMaps.registerVector({Add0}, &A);
+ SmallPtrSet<sandboxir::Instruction *, 4> Claimed;
+ EXPECT_TRUE(
+ sandboxir::VecUtils::getNextUserBundles({Ld0, Ld1}, IMaps, Claimed)
+ .empty());
}
More information about the llvm-branch-commits
mailing list