[llvm] [SandboxVectorizer] Implement topdown/bottomup vectorizers in unison (PR #205249)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 10:00:44 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/205249
>From 1ec2a4b9e65423a1da1dccd1b455a4e7d23aaeaa Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Sun, 14 Jun 2026 17:01:48 -0400
Subject: [PATCH] [SBVec] Implement topDown/botUp vectorizers in unison
This patch introduces the `top-down-vec` pass to the Sandbox Vectorizer,
adding the ability to traverse use-def chains top-down to discover and
collect vectorization opportunities. Furthermore, this patch unifies
the two vectorizers into a single implementation to minimize code
duplication.
---
.../SandboxVectorizer/Passes/BottomUpVec.h | 33 +-
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 179 +++++++++--
.../SandboxVectorizer/Passes/PassRegistry.def | 1 +
.../SandboxVectorizer/external_uses.ll | 52 ++-
.../test/Transforms/SandboxVectorizer/pack.ll | 79 ++++-
.../SandboxVectorizer/topdown_vec.ll | 304 ++++++++++++++++++
6 files changed, 610 insertions(+), 38 deletions(-)
create mode 100644 llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index d024bb74ac0c9..72264595ed0e3 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -6,7 +6,8 @@
//
//===----------------------------------------------------------------------===//
//
-// A Bottom-Up Vectorizer pass.
+// A bottom-up vectorizer pass. TopDownVec reuses this implementation with a
+// different traversal direction.
//
#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
@@ -22,6 +23,8 @@
namespace llvm::sandboxir {
+enum class VecDirection { TopDown, BottomUp };
+
/// This is a simple bottom-up vectorizer Region pass.
/// It expects a "seed slice" as an input in the Region's Aux vector.
/// The "seed slice" is a vector of instructions that can be used as a starting
@@ -32,7 +35,10 @@ namespace llvm::sandboxir {
/// profitable or not. For now profitability is checked at the end of the region
/// pass pipeline by a dedicated pass that accepts or rejects the IR
/// transaction, depending on the cost.
-class LLVM_ABI BottomUpVec final : public RegionPass {
+class LLVM_ABI BottomUpVec : public RegionPass {
+protected:
+ VecDirection Direction = VecDirection::BottomUp;
+ /// Set to true whenever the pass emits vector code in the current region.
bool Change = false;
/// The original instructions that are potentially dead after vectorization.
DenseSet<Instruction *> DeadInstrCandidates;
@@ -61,6 +67,10 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
/// for loads/stores) so that they can be cleaned up later.
void collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl);
+ StringRef vecDirectionToStr() {
+ return Direction == VecDirection::TopDown ? "TopDownVec" : "BottomUpVec";
+ }
+
/// Helper class describing how(if) to vectorize the code.
class ActionsVector {
private:
@@ -84,8 +94,11 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
/// vectorize in vectorizeRec().
unsigned DebugBndlCnt = 0;
- /// Recursively try to vectorize \p Bndl and its operands. This populates the
- /// `Actions` vector.
+ explicit BottomUpVec(StringRef Name, VecDirection Dir)
+ : RegionPass(Name), Direction(Dir) {}
+
+ /// Recursively try to vectorize \p Bndl. For bottom-up vectorization \p
+ /// UserBndl tracks the bundle of users that led to this recursion.
Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
unsigned Depth, LegalityAnalysis &Legality);
/// If the values in \p Bndl have external users, then emit unpacks and
@@ -98,8 +111,16 @@ class LLVM_ABI BottomUpVec final : public RegionPass {
bool tryVectorize(ArrayRef<Value *> Seeds, LegalityAnalysis &Legality);
public:
- BottomUpVec() : RegionPass("bottom-up-vec") {}
- bool runOnRegion(Region &Rgn, const Analyses &A) final;
+ BottomUpVec() : BottomUpVec("bottom-up-vec", VecDirection::BottomUp) {}
+
+ bool runOnRegion(Region &Rgn, const Analyses &A) override;
+};
+
+/// Top-down vectorizer Region pass. It expects a "seed slice" in the Region's
+/// Aux vector and walks down the def-use chain from the seed instructions.
+class LLVM_ABI TopDownVec final : public BottomUpVec {
+public:
+ TopDownVec() : BottomUpVec("top-down-vec", VecDirection::TopDown) {}
};
} // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 6bf257fcf8b1d..ba3f4599f4129 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -13,6 +13,7 @@
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
#include "llvm/SandboxIR/Utils.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Debug.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
@@ -34,6 +35,7 @@ static cl::opt<unsigned long>
static constexpr unsigned long StopBundleDisabled =
std::numeric_limits<unsigned long>::max();
+
static cl::opt<unsigned long>
StopBundle("sbvec-stop-bndl", cl::init(StopBundleDisabled), cl::Hidden,
cl::desc("Vectorize up to this many bundles."));
@@ -278,6 +280,58 @@ void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
}
}
+/// From a user \p U0 of lane 0 (\p V0), try to form a bundle of matching users
+/// for all lanes in \p Bndl. Used by the top-down vectorizer only. Returns an
+/// empty vector if no complete bundle can be formed.
+static SmallVector<Value *, 4> getNextUserBundle(ArrayRef<Value *> Bndl,
+ User *U0, Value *V0,
+ InstrMaps &IMaps) {
+ auto *UI0 = dyn_cast<Instruction>(U0);
+ if (!UI0 || IMaps.isVectorized(UI0))
+ return {};
+
+ // Find the operand index at which U0 uses lane 0.
+ unsigned OpIdx = UI0->getNumOperands();
+ for (unsigned Idx : seq<unsigned>(UI0->getNumOperands())) {
+ if (UI0->getOperand(Idx) == V0) {
+ OpIdx = Idx;
+ break;
+ }
+ }
+ if (OpIdx == UI0->getNumOperands())
+ return {};
+
+ // Find a distinct matching user for each of the remaining lanes.
+ SmallVector<Value *, 4> NextUserBndl;
+ NextUserBndl.push_back(UI0);
+ SmallPtrSet<Instruction *, 4> Claimed;
+ Claimed.insert(UI0);
+ for (Value *V : drop_begin(Bndl)) {
+ Instruction *Match = nullptr;
+ for (User *U : V->users()) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || IMaps.isVectorized(UI) || Claimed.contains(UI))
+ continue;
+ if (UI->getOpcode() != UI0->getOpcode() ||
+ UI->getType() != UI0->getType())
+ continue;
+ // The whole bundle must live in the same block.
+ if (UI->getParent() != UI0->getParent())
+ continue;
+ // The user must consume this lane at the same operand index.
+ if (OpIdx >= UI->getNumOperands() || UI->getOperand(OpIdx) != V)
+ continue;
+ Match = UI;
+ break;
+ }
+ if (!Match)
+ return {};
+ Claimed.insert(Match);
+ NextUserBndl.push_back(Match);
+ }
+ return NextUserBndl;
+}
+
Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
ArrayRef<Value *> UserBndl, unsigned Depth,
LegalityAnalysis &Legality) {
@@ -285,9 +339,55 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
DebugBndlCnt++ >= StopBundle && StopBundle != StopBundleDisabled;
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
VecUtils::dump(Bndl));
- const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
- : Legality.canVectorize(Bndl);
+ const auto &LegalityRes =
+ StopForDebug ? Legality.getForcedPackForDebugging()
+ : Legality.canVectorize(Bndl,
+ /*SkipScheduling=*/Direction ==
+ VecDirection::TopDown);
LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
+
+ if (Direction == VecDirection::TopDown) {
+ auto ActionPtr = std::make_unique<Action>(&LegalityRes, Bndl,
+ ArrayRef<Value *>(), Depth);
+ Action *Action = ActionPtr.get();
+ if (LegalityRes.getSubclassID() == LegalityResultID::Widen)
+ IMaps->registerVector(Bndl, Action);
+
+ // Pre-order push so defs are before uses.
+ Actions.push_back(std::move(ActionPtr));
+ switch (LegalityRes.getSubclassID()) {
+ case LegalityResultID::Widen: {
+ // Walk down the def-use chain. Each lane in \p Bndl may feed several
+ // users, so we form every compatible user bundle and recurse into each
+ // one. A user bundle is compatible only if all of its users share the
+ // same opcode and type, live in the same block, are distinct and not
+ // already vectorized, and consume their corresponding element of \p Bndl
+ // at the same operand index, so that the widened vector lines up as a
+ // single vector operand.
+ //
+ // Recursing right after forming each bundle marks its instructions as
+ // vectorized (pre-order registration), which prevents sibling bundles
+ // from claiming the same instruction and guarantees termination.
+ Value *V0 = Bndl[0];
+ for (User *U0 : V0->users()) {
+ SmallVector<Value *, 4> NextUserBndl =
+ getNextUserBundle(Bndl, U0, V0, *IMaps);
+ if (NextUserBndl.size() == Bndl.size())
+ vectorizeRec(NextUserBndl, Bndl, Depth + 1, Legality);
+ }
+ break;
+ }
+ case LegalityResultID::DiamondReuse:
+ case LegalityResultID::DiamondReuseMultiInput:
+ case LegalityResultID::DiamondReuseWithShuffle:
+ case LegalityResultID::Pack:
+ llvm_unreachable("Not implemented.");
+ }
+
+ return Action;
+ }
+
+ // Bottom up direction
auto ActionPtr =
std::make_unique<Action>(&LegalityRes, Bndl, UserBndl, Depth);
SmallVector<Action *> Operands;
@@ -362,13 +462,27 @@ void BottomUpVec::emitUnpacksForExternalUses(const ArrayRef<Value *> Bndl,
}
for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
+ // Collect the distinct external users first. We can't redirect uses while
+ // iterating Elm's use list, as that would invalidate the iterator.
+ SmallVector<User *, 4> ExternalUsers;
+ SmallPtrSet<User *, 4> Seen;
for (User *U : Elm->users()) {
- // Skip users that we just vectorized.
+ // Skip users that we just vectorized. Note: we must only redirect the
+ // external (non-vectorized) uses to an unpack and leave the vectorized
+ // users untouched. A blanket replaceAllUsesWith() would also rewrite the
+ // operands of users we are going to vectorize but have not emitted yet
+ // (in the top-down direction a user bundle is emitted after its operand
+ // bundle), which would corrupt those operands.
if (IMaps->isVectorized(U))
continue;
- auto *LastUnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
- Elm->replaceAllUsesWith(LastUnpackV);
+ if (Seen.insert(U).second)
+ ExternalUsers.push_back(U);
}
+ if (ExternalUsers.empty())
+ continue;
+ auto *UnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
+ for (User *U : ExternalUsers)
+ U->replaceUsesOfWith(Elm, UnpackV);
}
}
@@ -387,22 +501,44 @@ Value *BottomUpVec::emitVectors() {
case LegalityResultID::Widen: {
auto *I = cast<Instruction>(Bndl[0]);
SmallVector<Value *, 2> VecOperands;
- switch (I->getOpcode()) {
- case Instruction::Opcode::Load:
- VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
- break;
- case Instruction::Opcode::Store: {
- VecOperands.push_back(ActionPtr->Operands[0]->Vec);
- VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
- break;
- }
- default:
- // Visit all operands.
- for (Action *OpA : ActionPtr->Operands) {
- auto *VecOp = OpA->Vec;
- VecOperands.push_back(VecOp);
+ if (Direction == VecDirection::BottomUp) {
+ switch (I->getOpcode()) {
+ case Instruction::Opcode::Load:
+ VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+ break;
+ case Instruction::Opcode::Store:
+ VecOperands.push_back(ActionPtr->Operands[0]->Vec);
+ VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+ break;
+ default:
+ for (Action *OpA : ActionPtr->Operands)
+ VecOperands.push_back(OpA->Vec);
+ break;
+ }
+ } else {
+ switch (I->getOpcode()) {
+ case Instruction::Opcode::Load:
+ VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
+ break;
+ case Instruction::Opcode::Store: {
+ auto OpBndl = getOperand(Bndl, 0);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+ VecOperands.push_back(OpA->Vec);
+ else
+ VecOperands.push_back(createPack(OpBndl, UserBB));
+ VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
+ break;
+ }
+ default:
+ for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) {
+ SmallVector<Value *, 4> OpBndl = getOperand(Bndl, OpIdx);
+ if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
+ VecOperands.push_back(OpA->Vec);
+ else
+ VecOperands.push_back(createPack(OpBndl, UserBB));
+ }
+ break;
}
- break;
}
NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
// Collect any potentially dead scalar instructions, including the
@@ -526,7 +662,8 @@ bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl,
Actions.clear();
DebugBndlCnt = 0;
vectorizeRec(Bndl, {}, /*Depth=*/0, Legality);
- LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "BottomUpVec: Vectorization Actions:\n";
+ LLVM_DEBUG(dbgs() << DEBUG_PREFIX << vecDirectionToStr()
+ << ": Vectorization Actions:\n";
Actions.dump());
emitVectors();
tryEraseDeadInstrs();
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index 10ba595910ee9..40d03c3886c0b 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -26,6 +26,7 @@ REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
REGION_PASS("tr-revert", ::llvm::sandboxir::TransactionAlwaysRevert)
REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
+REGION_PASS("top-down-vec", ::llvm::sandboxir::TopDownVec)
REGION_PASS("load-store-vec", ::llvm::sandboxir::LoadStoreVec)
#undef REGION_PASS
diff --git a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
index 593965ab01680..0bd6e9eebe5af 100644
--- a/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/external_uses.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-always-verify -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-collect-seeds=loads -sbvec-always-verify -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
; Checks the handling of users outside the vectorized graph.
@@ -13,6 +14,16 @@ define void @external_users(ptr %ptr) {
; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
; CHECK-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: [[VEC:%.*]] = fsub <2 x float> [[VECL]], zeroinitializer, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: [[SUB0:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg float [[SUB0]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -35,6 +46,17 @@ define void @external_user_of_constant(ptr %ptr, ptr %ptrX) {
; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
; CHECK-NEXT: store i32 0, ptr [[PTRX]], align 4
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @external_user_of_constant(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], ptr [[PTRX:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[ZEXT0:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: [[ZEXT1:%.*]] = zext i16 0 to i32
+; TOPDOWN-NEXT: store i32 [[ZEXT0]], ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: store i32 [[ZEXT1]], ptr [[PTR1]], align 4
+; TOPDOWN-NEXT: store i32 [[ZEXT0]], ptr [[PTRX]], align 4
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -57,6 +79,20 @@ define void @vector_external_users(ptr %ptr) {
; CHECK-NEXT: store <3 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META2]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACKINS2]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; TOPDOWN-NEXT: [[VECL:%.*]] = load <3 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[LD0:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: [[LD1:%.*]] = shufflevector <3 x float> [[VECL]], <3 x float> poison, <2 x i32> <i32 1, i32 2>, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: [[SUB0:%.*]] = fsub float [[LD0]], 0.000000e+00
+; TOPDOWN-NEXT: [[SUB1:%.*]] = fsub <2 x float> [[LD1]], zeroinitializer
+; TOPDOWN-NEXT: store float [[SUB0]], ptr [[PTR0]], align 4
+; TOPDOWN-NEXT: store <2 x float> [[SUB1]], ptr [[PTR1]], align 8
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr float, ptr %ptr, i32 0
%ptr1 = getelementptr float, ptr %ptr, i32 1
@@ -80,6 +116,16 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK-NEXT: store <4 x float> [[VEC]], ptr [[PTR0]], align 8, !sandboxvec [[META3]]
; CHECK-NEXT: [[USER:%.*]] = fneg <2 x float> [[UNPACK]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @vector_external_users_lane_and_index_differ(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr <2 x float>, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: [[VECL:%.*]] = load <4 x float>, ptr [[PTR0]], align 8, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VEC:%.*]] = fsub <4 x float> [[VECL]], zeroinitializer, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[SUB1:%.*]] = shufflevector <4 x float> [[VEC]], <4 x float> poison, <2 x i32> <i32 2, i32 3>, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: store <4 x float> [[VEC]], ptr [[PTR0]], align 8, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[USER:%.*]] = fneg <2 x float> [[SUB1]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr <2 x float>, ptr %ptr, i32 0
%ptr1 = getelementptr <2 x float>, ptr %ptr, i32 1
@@ -99,3 +145,7 @@ define void @vector_external_users_lane_and_index_differ(ptr %ptr) {
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index 743d705fd48ff..6a91f3cdb5d24 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-always-verify -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-always-verify -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=TOPDOWN
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
@@ -7,6 +8,12 @@ define void @pack_constants(ptr %ptr) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
; CHECK-NEXT: ret void
+;
+; TOPDOWN-LABEL: define void @pack_constants(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
+; TOPDOWN-NEXT: store <2 x i8> <i8 0, i8 1>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
+; TOPDOWN-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i32 0
%ptr1 = getelementptr i8, ptr %ptr, i32 1
@@ -27,14 +34,31 @@ define void @packPHIs(ptr %ptr) {
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packPHIs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI2:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI3:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[PHI0]], i32 0, !sandboxvec [[META1:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[PHI1]], i32 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META1]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
br label %loop
@@ -63,14 +87,31 @@ define void @packFromOtherBB(ptr %ptr, i8 %val) {
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT:.*:]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromOtherBB(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[VAL:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*]]:
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[VAL]], 0
+; TOPDOWN-NEXT: [[MUL1:%.*]] = mul i8 [[VAL]], 1
+; TOPDOWN-NEXT: br label %[[LOOP:.*]]
+; TOPDOWN: [[LOOP]]:
+; TOPDOWN-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META2:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[MUL1]], i32 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META2]]
+; TOPDOWN-NEXT: br label %[[LOOP]]
+; TOPDOWN: [[EXIT:.*:]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %val, 0
%mul1 = mul i8 %val, 1
@@ -97,12 +138,25 @@ define void @packFromDiffBBs(ptr %ptr, i8 %v) {
; CHECK-NEXT: br label %[[BB:.*]]
; CHECK: [[BB]]:
; CHECK-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
-; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
-; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; CHECK-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; CHECK-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
; CHECK-NEXT: ret void
;
+; TOPDOWN-LABEL: define void @packFromDiffBBs(
+; TOPDOWN-SAME: ptr [[PTR:%.*]], i8 [[V:%.*]]) {
+; TOPDOWN-NEXT: [[ENTRY:.*:]]
+; TOPDOWN-NEXT: [[ADD0:%.*]] = add i8 [[V]], 1
+; TOPDOWN-NEXT: br label %[[BB:.*]]
+; TOPDOWN: [[BB]]:
+; TOPDOWN-NEXT: [[ADD1:%.*]] = add i8 [[V]], 2
+; TOPDOWN-NEXT: [[VPACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0, !sandboxvec [[META3:![0-9]+]]
+; TOPDOWN-NEXT: [[VPACK1:%.*]] = insertelement <2 x i8> [[VPACK]], i8 [[ADD1]], i32 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; TOPDOWN-NEXT: store <2 x i8> [[VPACK1]], ptr [[GEP0]], align 1, !sandboxvec [[META3]]
+; TOPDOWN-NEXT: ret void
+;
entry:
%add0 = add i8 %v, 1
br label %bb
@@ -121,3 +175,8 @@ bb:
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
;.
+; TOPDOWN: [[META0]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META1]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META2]] = distinct !{!"sandboxregion"}
+; TOPDOWN: [[META3]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
new file mode 100644
index 0000000000000..1f2b697c3b5b0
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/topdown_vec.ll
@@ -0,0 +1,304 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 \
+; RUN: -sbvec-collect-seeds=loads -sbvec-always-verify \
+; RUN: -sbvec-passes="seed-collection<tr-save,top-down-vec,tr-accept>" \
+; RUN: %s -S | FileCheck %s
+
+; Tests: successful bundle match (baseline) and !IMaps->isVectorized(UI0) on
+; the outer loop's second use edge (%ld0 used twice by the same fadd).
+define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_fadd_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META0]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META0]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define void @load_chain_store(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @load_chain_store(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fmul <2 x float> [[VECL]], splat (float 3.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[VEC1:%.*]] = fadd <2 x float> [[VEC]], splat (float 2.000000e+00), !sandboxvec [[META1]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC1]], ptr [[PTR2_0]], align 4, !sandboxvec [[META1]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fmul0 = fmul float %ld0, 3.0
+ %fmul1 = fmul float %ld1, 3.0
+
+ %fadd0 = fadd float %fmul0, 2.0
+ %fadd1 = fadd float %fmul1, 2.0
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+define float @load_fadd_external_use(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define float @load_fadd_external_use(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META2]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_0]], align 4, !sandboxvec [[META2]]
+; CHECK-NEXT: ret float [[UNPACK]]
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fadd1 = fadd float %ld1, %ld1
+
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd0, ptr %ptr2_0
+ store float %fadd1, ptr %ptr2_1
+
+ ret float %fadd0
+}
+
+; Both lanes feed the *same* user instruction (once per operand). The user
+; bundle must not be formed out of duplicate instructions, so the fadd stays
+; scalar while the loads still widen.
+define float @single_user_no_duplicate(ptr %ptr) {
+; CHECK-LABEL: define float @single_user_no_duplicate(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META3]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META3]]
+; CHECK-NEXT: [[FADD:%.*]] = fadd float [[UNPACK]], [[UNPACK1]]
+; CHECK-NEXT: ret float [[FADD]]
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+
+ %fadd = fadd float %ld0, %ld1
+ ret float %fadd
+}
+
+; The candidate users live in different blocks from each other, so they must
+; not be bundled together.
+define void @users_in_different_blocks(ptr %ptr, ptr %ptr2, i1 %c) {
+; CHECK-LABEL: define void @users_in_different_blocks(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META4:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META4]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META4]]
+; CHECK-NEXT: br i1 [[C]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK: [[BB0]]:
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[BB1]]:
+; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: store float [[FADD1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+ br i1 %c, label %if.then, label %if.else
+
+if.then:
+ %fadd0 = fadd float %ld0, %ld0
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ store float %fadd0, ptr %ptr2_0
+ ret void
+
+if.else:
+ %fadd1 = fadd float %ld1, %ld1
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ store float %fadd1, ptr %ptr2_1
+ ret void
+}
+
+; Lane 0 feeds fadd, lane 1 feeds fmul — opcode mismatch rejects the bundle.
+define void @user_opcode_mismatch(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_opcode_mismatch(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META5:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META5]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META5]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FMUL1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd float %ld0, %ld0
+ %fmul1 = fmul float %ld1, %ld1
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store float %fmul1, ptr %ptr2_1, align 4
+ ret void
+}
+
+; Lane 0's user is fadd float, lane 1's user is fadd double — type mismatch.
+define void @user_type_mismatch(ptr %ptr, ptr %ptr2) {
+; CHECK-LABEL: define void @user_type_mismatch(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr double, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META6:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META6]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META6]]
+; CHECK-NEXT: [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
+; CHECK-NEXT: [[EXT1:%.*]] = fpext float [[UNPACK1]] to double
+; CHECK-NEXT: [[FADD1:%.*]] = fadd double [[EXT1]], [[EXT1]]
+; CHECK-NEXT: store float [[FADD0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store double [[FADD1]], ptr [[PTR2_1]], align 8
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr double, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd float %ld0, %ld0
+ %ext1 = fpext float %ld1 to double
+ %fadd1 = fadd double %ext1, %ext1
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store double %fadd1, ptr %ptr2_1, align 8
+ ret void
+}
+
+; Lane 0 uses ld0 at operand 0; lane 1 uses ld1 at operand 1 (fsub is not
+; commutative). Operand-index mismatch rejects the bundle.
+define void @user_operand_index_mismatch(ptr %ptr, ptr %ptr2, float %x) {
+; CHECK-LABEL: define void @user_operand_index_mismatch(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[X:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_1:%.*]] = getelementptr float, ptr [[PTR2]], i32 1
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META7:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META7]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META7]]
+; CHECK-NEXT: [[FSUB0:%.*]] = fsub float [[UNPACK]], [[X]]
+; CHECK-NEXT: [[FSUB1:%.*]] = fsub float [[X]], [[UNPACK1]]
+; CHECK-NEXT: store float [[FSUB0]], ptr [[PTR2_0]], align 4
+; CHECK-NEXT: store float [[FSUB1]], ptr [[PTR2_1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fsub0 = fsub float %ld0, %x
+ %fsub1 = fsub float %x, %ld1
+
+ store float %fsub0, ptr %ptr2_0, align 4
+ store float %fsub1, ptr %ptr2_1, align 4
+ ret void
+}
+
+; ld0 has two fadd users; the first bundle {fadd0,fadd1} vectorizes fadd1.
+; When matching the second bundle, fadd1 is skipped (already vectorized) and
+; fadd1b is chosen instead.
+define void @user_already_vectorized(ptr %ptr, ptr %ptr2, float %a, float %b) {
+; CHECK-LABEL: define void @user_already_vectorized(
+; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META8:![0-9]+]]
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META8]]
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META8]]
+; CHECK-NEXT: [[PTR2_0:%.*]] = getelementptr float, ptr [[PTR2]], i32 0
+; CHECK-NEXT: [[PTR2_2:%.*]] = getelementptr float, ptr [[PTR2]], i32 2
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META8]]
+; CHECK-NEXT: [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META8]]
+; CHECK-NEXT: [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META8]]
+; CHECK-NEXT: store <2 x float> [[VEC4]], ptr [[PTR2_0]], align 4, !sandboxvec [[META8]]
+; CHECK-NEXT: store <2 x float> [[VEC]], ptr [[PTR2_2]], align 4, !sandboxvec [[META8]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
+ %ptr2_1 = getelementptr float, ptr %ptr2, i32 1
+ %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
+ %ptr2_3 = getelementptr float, ptr %ptr2, i32 3
+
+ %ld0 = load float, ptr %ptr0, align 4
+ %ld1 = load float, ptr %ptr1, align 4
+
+ %fadd0 = fadd float %ld0, %a
+ %fadd0b = fadd float %ld0, %b
+ %fadd1 = fadd float %ld1, %a
+ %fadd1b = fadd float %ld1, %b
+
+ store float %fadd0, ptr %ptr2_0, align 4
+ store float %fadd1, ptr %ptr2_1, align 4
+ store float %fadd0b, ptr %ptr2_2, align 4
+ store float %fadd1b, ptr %ptr2_3, align 4
+ ret void
+}
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+; CHECK: [[META2]] = distinct !{!"sandboxregion"}
+; CHECK: [[META3]] = distinct !{!"sandboxregion"}
+; CHECK: [[META4]] = distinct !{!"sandboxregion"}
+; CHECK: [[META5]] = distinct !{!"sandboxregion"}
+; CHECK: [[META6]] = distinct !{!"sandboxregion"}
+; CHECK: [[META7]] = distinct !{!"sandboxregion"}
+; CHECK: [[META8]] = distinct !{!"sandboxregion"}
+;.
More information about the llvm-commits
mailing list