[llvm] [SandboxVec][BottomUpVec] Add cost estimation and tr-accept-or-revert pass (PR #124620)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 7 15:48:53 PST 2025
https://github.com/vporpo updated https://github.com/llvm/llvm-project/pull/124620
>From f1594cd87872418cfa61ecb1a9a932f307cb716b Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas at google.com>
Date: Fri, 24 Jan 2025 15:55:40 -0800
Subject: [PATCH 1/4] [SandboxVec][BottomUpVec] Add cost estimation and
tr-accept-or-revert pass
The TransactionAcceptOrRevert pass is the final pass in the Sandbox Vectorizer's
default pass pipeline. It's job is to check the cost before/after vectorization
and accept or revert the IR to its original state.
Since we are now starting the transaction in BottomUpVec, tests that run a
custom pipeline need to accept the transaction. This is done with the help
of the TransactionAlwaysAccept pass (tr-accept).
---
llvm/include/llvm/SandboxIR/Tracker.h | 2 +
.../Passes/TransactionAcceptOrRevert.h | 30 ++++++++++
.../Passes/TransactionAlwaysAccept.h | 34 ++++++++++++
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 1 +
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 20 +++++--
.../SandboxVectorizer/Passes/PassRegistry.def | 2 +
.../Passes/TransactionAcceptOrRevert.cpp | 37 +++++++++++++
.../SandboxVectorizer/SandboxVectorizer.cpp | 5 +-
.../SandboxVectorizerPassBuilder.cpp | 2 +
.../SandboxVectorizer/X86/simple_cost_test.ll | 55 +++++++++++++++++++
.../SandboxVectorizer/bottomup_basic.ll | 2 +-
.../SandboxVectorizer/bottomup_seed_slice.ll | 2 +-
.../bottomup_seed_slice_pow2.ll | 4 +-
.../Transforms/SandboxVectorizer/cross_bbs.ll | 2 +-
.../default_pass_pipeline.ll | 2 +
.../test/Transforms/SandboxVectorizer/pack.ll | 2 +-
llvm/unittests/SandboxIR/TrackerTest.cpp | 6 ++
17 files changed, 196 insertions(+), 12 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h
create mode 100644 llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
create mode 100644 llvm/test/Transforms/SandboxVectorizer/X86/simple_cost_test.ll
diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 9a031f327083740..94b259722bbada9 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -473,6 +473,8 @@ class Tracker {
~Tracker();
Context &getContext() const { return Ctx; }
+ /// \Returns true if there are no changes tracked.
+ bool empty() const { return Changes.empty(); }
/// Record \p Change and take ownership. This is the main function used to
/// track Sandbox IR changes.
void track(std::unique_ptr<IRChangeBase> &&Change) {
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h
new file mode 100644
index 000000000000000..fce9cc0c1bde778
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h
@@ -0,0 +1,30 @@
+//===- TransactionAcceptOrRevert.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a region pass that checks the region cost before/after vectorization
+// and accepts the state of Sandbox IR if the cost is better, or otherwise
+// reverts it.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONACCEPTORREVERT_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONACCEPTORREVERT_H
+
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/Region.h"
+
+namespace llvm::sandboxir {
+
+class TransactionAcceptOrRevert : public RegionPass {
+public:
+ TransactionAcceptOrRevert() : RegionPass("tr-accept-or-revert") {}
+ bool runOnRegion(Region &Rgn, const Analyses &A) final;
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONACCEPTORREVERT_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h
new file mode 100644
index 000000000000000..ed6cf1bf7cf51e5
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h
@@ -0,0 +1,34 @@
+//===- TransactionAlwaysAccept.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a region pass that always accepts the transaction without checking
+// its cost. This is mainly used as a final pass in lit tests.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONALWAYSACCEPT_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONALWAYSACCEPT_H
+
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/Region.h"
+
+namespace llvm::sandboxir {
+
+class TransactionAlwaysAccept : public RegionPass {
+public:
+ TransactionAlwaysAccept() : RegionPass("tr-accept") {}
+ bool runOnRegion(Region &Rgn, const Analyses &A) final {
+ auto &Tracker = Rgn.getContext().getTracker();
+ bool HasChanges = !Tracker.empty();
+ Tracker.accept();
+ return HasChanges;
+ }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONALWAYSACCEPT_H
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index e5fabd318b82cc7..872e055294d5574 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/Legality.cpp
SandboxVectorizer/Passes/BottomUpVec.cpp
SandboxVectorizer/Passes/RegionsFromMetadata.cpp
+ SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
SandboxVectorizer/SandboxVectorizer.cpp
SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
SandboxVectorizer/Scheduler.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 6f65657d297906b..507d16324012700 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -12,6 +12,7 @@
#include "llvm/SandboxIR/Function.h"
#include "llvm/SandboxIR/Instruction.h"
#include "llvm/SandboxIR/Module.h"
+#include "llvm/SandboxIR/Region.h"
#include "llvm/SandboxIR/Utils.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
@@ -448,13 +449,24 @@ bool BottomUpVec::runOnFunction(Function &F, const Analyses &A) {
assert(SeedSlice.size() >= 2 && "Should have been rejected!");
- // TODO: If vectorization succeeds, run the RegionPassManager on the
- // resulting region.
-
// TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
SmallVector<Value *> SeedSliceVals(SeedSlice.begin(),
SeedSlice.end());
- Change |= tryVectorize(SeedSliceVals);
+ // Create an empty region. Instructions get added to the region
+ // automatically by the callbacks.
+ auto &Ctx = F.getContext();
+ Region Rgn(Ctx, A.getTTI());
+ // Save the state of the IR before we make any changes. The
+ // transaction gets accepted/reverted by the tr-accept-or-revert pass.
+ Ctx.save();
+ // Try to vectorize starting from the seed slice. The returned value
+ // is true if we found vectorizable code and generated some vector
+ // code for it. It does not mean that the code is profitable.
+ bool VecSuccess = tryVectorize(SeedSliceVals);
+ if (VecSuccess)
+ // WARNING: All passes should return false, except those that
+ // accept/revert the state.
+ Change |= RPM.runOnRegion(Rgn, A);
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index 0dc72842f1abe0e..f3aa12729860ff6 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -19,6 +19,8 @@
REGION_PASS("null", ::llvm::sandboxir::NullPass)
REGION_PASS("print-instruction-count", ::llvm::sandboxir::PrintInstructionCount)
+REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
+REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
#undef REGION_PASS
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
new file mode 100644
index 000000000000000..73baace002a3229
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
@@ -0,0 +1,37 @@
+//===- TransactionAcceptOrRevert.cpp - Check cost and accept/revert region ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InstructionCost.h"
+
+namespace llvm {
+
+static cl::opt<bool> CostThreshold("sbvec-cost-threshold", cl::init(0),
+ cl::Hidden,
+ cl::desc("Vectorization cost threshold."));
+
+namespace sandboxir {
+
+bool TransactionAcceptOrRevert::runOnRegion(Region &Rgn, const Analyses &A) {
+ const auto &SB = Rgn.getScoreboard();
+ InstructionCost CostAfterMinusBefore = SB.getAfterCost() - SB.getBeforeCost();
+ // TODO: Print costs / write to remarks.
+ auto &Tracker = Rgn.getContext().getTracker();
+ if (CostAfterMinusBefore < -CostThreshold) {
+ bool HasChanges = !Tracker.empty();
+ Tracker.accept();
+ return HasChanges;
+ }
+ // Revert the IR.
+ Rgn.getContext().getTracker().revert();
+ return false;
+}
+
+} // namespace sandboxir
+} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
index 798a0ad915375bc..b233d35212f9471 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
@@ -31,9 +31,10 @@ static cl::opt<std::string> UserDefinedPassPipeline(
SandboxVectorizerPass::SandboxVectorizerPass() : FPM("fpm") {
if (UserDefinedPassPipeline == DefaultPipelineMagicStr) {
- // TODO: Add region passes to the default pipeline.
+ // TODO: Add passes to the default pipeline. It currently contains:
+ // - the bottom-up-vectorizer pass
FPM.setPassPipeline(
- "bottom-up-vec<>",
+ "bottom-up-vec<tr-accept-or-revert>",
sandboxir::SandboxVectorizerPassBuilder::createFunctionPass);
} else {
// Create the user-defined pipeline.
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 5ecf7b2ed0d258e..0c1ab55e91a5cfc 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -4,6 +4,8 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
namespace llvm::sandboxir {
diff --git a/llvm/test/Transforms/SandboxVectorizer/X86/simple_cost_test.ll b/llvm/test/Transforms/SandboxVectorizer/X86/simple_cost_test.ll
new file mode 100644
index 000000000000000..2db50ce582f8343
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/X86/simple_cost_test.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -mtriple=x86_64-- -mattr=+sse4.1 %s -S | FileCheck %s
+
+define void @simple_cost_test(ptr %ptr) {
+; CHECK-LABEL: define void @simple_cost_test(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr double, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECL:%.*]] = load <2 x double>, ptr [[PTR0]], align 8
+; CHECK-NEXT: store <2 x double> [[VECL]], ptr [[PTR0]], align 8
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr double, ptr %ptr, i32 0
+ %ptr1 = getelementptr double, ptr %ptr, i32 1
+ %ld0 = load double, ptr %ptr0
+ %ld1 = load double, ptr %ptr1
+ store double %ld0, ptr %ptr0
+ store double %ld1, ptr %ptr1
+ ret void
+}
+
+define void @pack_cost_test_(ptr %ptr) {
+; CHECK-LABEL: define void @pack_cost_test_(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR1:%.*]] = getelementptr float, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[LD0:%.*]] = load float, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
+; CHECK-NEXT: [[PACK4:%.*]] = insertelement <4 x float> poison, float [[LD0]], i32 0
+; CHECK-NEXT: [[PACK5:%.*]] = insertelement <4 x float> [[PACK4]], float [[LD1]], i32 1
+; CHECK-NEXT: [[PACK6:%.*]] = insertelement <4 x float> [[PACK5]], float [[LD0]], i32 2
+; CHECK-NEXT: [[PACK7:%.*]] = insertelement <4 x float> [[PACK6]], float [[LD1]], i32 3
+; CHECK-NEXT: [[PACK:%.*]] = insertelement <4 x float> poison, float [[LD0]], i32 0
+; CHECK-NEXT: [[PACK1:%.*]] = insertelement <4 x float> [[PACK]], float [[LD1]], i32 1
+; CHECK-NEXT: [[PACK2:%.*]] = insertelement <4 x float> [[PACK1]], float [[LD0]], i32 2
+; CHECK-NEXT: [[PACK3:%.*]] = insertelement <4 x float> [[PACK2]], float [[LD1]], i32 3
+; CHECK-NEXT: [[VEC:%.*]] = fmul <4 x float> [[PACK3]], [[PACK7]]
+; CHECK-NEXT: store <4 x float> [[VEC]], ptr [[PTR0]], align 4
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr float, ptr %ptr, i32 0
+ %ptr1 = getelementptr float, ptr %ptr, i32 1
+ %ptr2 = getelementptr float, ptr %ptr, i32 2
+ %ptr3 = getelementptr float, ptr %ptr, i32 3
+ %ld0 = load float, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+ %mul0 = fmul float %ld0, %ld0
+ %mul1 = fmul float %ld1, %ld1
+ %mul2 = fmul float %ld0, %ld0
+ %mul3 = fmul float %ld1, %ld1
+ store float %mul0, ptr %ptr0
+ store float %mul1, ptr %ptr1
+ store float %mul2, ptr %ptr2
+ store float %mul3, ptr %ptr3
+ ret void
+}
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
index ee5a3a514b3c58e..ee8592c04b62c0a 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
define void @store_load(ptr %ptr) {
; CHECK-LABEL: define void @store_load(
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
index 8459c3addaa83f1..202b5a6fbd6c968 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
declare void @foo()
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
index e186d5fa86e4a7f..f1c6e3297d79c7f 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s --check-prefix=POW2
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s --check-prefix=NON-POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
define void @pow2(ptr %ptr, float %val) {
; POW2-LABEL: define void @pow2(
diff --git a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
index 6ec31060d7e0fe7..ff1604173c31754 100644
--- a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
define void @cross_bbs(ptr %ptr) {
; CHECK-LABEL: define void @cross_bbs(
diff --git a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
index 1d7be43336c8795..10de4338caf2325 100644
--- a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
@@ -4,8 +4,10 @@
; This checks the default pass pipeline for the sandbox vectorizer.
define void @pipeline() {
+; CHECK: fpm
; CHECK: bottom-up-vec
; CHECK: rpm
+; CHECK: tr-accept-or-revert
; CHECK-EMPTY:
ret void
}
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index ec6e61a90c0fb3f..da41036e3a58b76 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index 4eedab124bfa047..9c18247b6b96d0e 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -52,6 +52,9 @@ define void @foo(ptr %ptr) {
auto *F = Ctx.createFunction(&LLVMF);
auto *BB = &*F->begin();
auto &Tracker = Ctx.getTracker();
+ // Check empty().
+ EXPECT_TRUE(Ctx.getTracker().empty());
+
Tracker.save();
auto It = BB->begin();
auto *Gep0 = &*It++;
@@ -65,6 +68,9 @@ define void @foo(ptr %ptr) {
EXPECT_EQ(St->getOperand(1), Gep1);
EXPECT_EQ(Ld->getOperand(0), Gep1);
+ // Check empty().
+ EXPECT_FALSE(Ctx.getTracker().empty());
+
Ctx.getTracker().revert();
EXPECT_NE(St->getOperand(0), Ld);
EXPECT_EQ(St->getOperand(1), Gep0);
>From cf23a78a585366fae892378f0df5148d36c1d5a8 Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas at google.com>
Date: Tue, 28 Jan 2025 17:01:54 -0800
Subject: [PATCH 2/4] [SandboxIR][Region] Implement an auxiliary vector in
Region
This patch adds additional functionality to the sandboxir Region.
The Region is used as a way of passing a set of Instructions across region
passes in a way that can be represented in the IR with metadata.
This is a design choice that allows us to test region passes in isolation
with lit tests.
Up until now the region was only used to tag the instructions generated
by the passes. There is a need to represent an ordered set of instructions,
which can be used as a way to represent the initial seeds to the first
vectorization pass. This patch implements this auxiliary vector that
can be used to convey such information.
---
llvm/include/llvm/SandboxIR/Region.h | 15 ++++
llvm/lib/SandboxIR/Region.cpp | 58 ++++++++++++++-
llvm/unittests/SandboxIR/RegionTest.cpp | 96 +++++++++++++++++++++++++
3 files changed, 166 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/SandboxIR/Region.h b/llvm/include/llvm/SandboxIR/Region.h
index c1195141cb54ce3..c81ae3c6b69ebb8 100644
--- a/llvm/include/llvm/SandboxIR/Region.h
+++ b/llvm/include/llvm/SandboxIR/Region.h
@@ -87,16 +87,23 @@ class ScoreBoard {
// | |
// |Rgn3| -> Transform1 -> ... -> TransformN -> Check Cost
// +----+
+//
+// The region can also hold an ordered sequence of "auxiliary" instructions.
+// This can be used to pass auxiliary information across region passes, like for
+// example the initial seed slice used by the bottom-up vectorizer.
class Region {
/// All the instructions in the Region. Only new instructions generated during
/// vectorization are part of the Region.
SetVector<Instruction *> Insts;
+ /// An auxiliary sequence of Instruction-Index pairs.
+ SmallVector<Instruction *> Aux;
/// MDNode that we'll use to mark instructions as being part of the region.
MDNode *RegionMDN;
static constexpr const char *MDKind = "sandboxvec";
static constexpr const char *RegionStr = "sandboxregion";
+ static constexpr const char *AuxMDKind = "sbaux";
Context &Ctx;
/// Keeps track of cost of instructions added and removed.
@@ -110,6 +117,10 @@ class Region {
// TODO: Add cost modeling.
// TODO: Add a way to encode/decode region info to/from metadata.
+ /// Set \p I as the \p Idx'th element in the auxiliary vector.
+ /// NOTE: This is for internal use, it does not set the metadata.
+ void setAux(uint32_t Idx, Instruction *I);
+
public:
Region(Context &Ctx, TargetTransformInfo &TTI);
~Region();
@@ -124,6 +135,10 @@ class Region {
bool contains(Instruction *I) const { return Insts.contains(I); }
/// Returns true if the Region has no instructions.
bool empty() const { return Insts.empty(); }
+ /// Set the auxiliary vector.
+ void setAux(ArrayRef<Instruction *> Aux);
+ /// \Returns the auxiliary vector.
+ const SmallVector<Instruction *> &getAux() const { return Aux; }
using iterator = decltype(Insts.begin());
iterator begin() { return Insts.begin(); }
diff --git a/llvm/lib/SandboxIR/Region.cpp b/llvm/lib/SandboxIR/Region.cpp
index dbb000e5dd92530..9f4437b7633dc62 100644
--- a/llvm/lib/SandboxIR/Region.cpp
+++ b/llvm/lib/SandboxIR/Region.cpp
@@ -57,6 +57,31 @@ void Region::add(Instruction *I) {
Scoreboard.add(I);
}
+void Region::setAux(ArrayRef<Instruction *> Aux) {
+ this->Aux = SmallVector<Instruction *>(Aux);
+ auto &LLVMCtx = Ctx.LLVMCtx;
+ for (auto [Idx, I] : enumerate(Aux)) {
+ llvm::ConstantInt *IdxC =
+ llvm::ConstantInt::get(LLVMCtx, llvm::APInt(32, Idx, false));
+ assert(cast<llvm::Instruction>(I->Val)->getMetadata(AuxMDKind) == nullptr &&
+ "Instruction already in Aux!");
+ cast<llvm::Instruction>(I->Val)->setMetadata(
+ AuxMDKind, MDNode::get(LLVMCtx, ConstantAsMetadata::get(IdxC)));
+ }
+}
+
+void Region::setAux(uint32_t Idx, Instruction *I) {
+ uint32_t ExpectedSz = Idx + 1;
+ if (Aux.size() < ExpectedSz) {
+ auto SzBefore = Aux.size();
+ Aux.resize(ExpectedSz);
+ // Initialize the gap with nullptr.
+ for (unsigned Idx = SzBefore; Idx + 1 < ExpectedSz; ++Idx)
+ Aux[Idx] = nullptr;
+ }
+ Aux[Idx] = I;
+}
+
void Region::remove(Instruction *I) {
// Keep track of the instruction cost. This need to be done *before* we remove
// `I` from the region.
@@ -78,6 +103,15 @@ bool Region::operator==(const Region &Other) const {
void Region::dump(raw_ostream &OS) const {
for (auto *I : Insts)
OS << *I << "\n";
+ if (!Aux.empty()) {
+ OS << "\nAux:\n";
+ for (auto *I : Aux) {
+ if (I == nullptr)
+ OS << "NULL\n";
+ else
+ OS << *I << "\n";
+ }
+ }
}
void Region::dump() const {
@@ -93,16 +127,34 @@ Region::createRegionsFromMD(Function &F, TargetTransformInfo &TTI) {
auto &Ctx = F.getContext();
for (BasicBlock &BB : F) {
for (Instruction &Inst : BB) {
- if (auto *MDN = cast<llvm::Instruction>(Inst.Val)->getMetadata(MDKind)) {
+ auto *LLVMI = cast<llvm::Instruction>(Inst.Val);
+ if (auto *MDN = LLVMI->getMetadata(MDKind)) {
+ Region *R = nullptr;
auto [It, Inserted] = MDNToRegion.try_emplace(MDN);
if (Inserted) {
Regions.push_back(std::make_unique<Region>(Ctx, TTI));
- It->second = Regions.back().get();
+ R = Regions.back().get();
+ It->second = R;
+ } else {
+ R = It->second;
+ }
+ R->add(&Inst);
+
+ if (auto *AuxMDN = LLVMI->getMetadata(AuxMDKind)) {
+ llvm::Constant *IdxC =
+ dyn_cast<ConstantAsMetadata>(AuxMDN->getOperand(0))->getValue();
+ auto Idx = cast<llvm::ConstantInt>(IdxC)->getSExtValue();
+ R->setAux(Idx, &Inst);
}
- It->second->add(&Inst);
}
}
}
+#ifndef NDEBUG
+ // Check that there are no gaps in the Aux vector.
+ for (auto &RPtr : Regions)
+ for (auto *I : RPtr->getAux())
+ assert(I != nullptr && "Gap in Aux!");
+#endif
return Regions;
}
diff --git a/llvm/unittests/SandboxIR/RegionTest.cpp b/llvm/unittests/SandboxIR/RegionTest.cpp
index 1ee72d127daa4d5..31dbf9661daa0be 100644
--- a/llvm/unittests/SandboxIR/RegionTest.cpp
+++ b/llvm/unittests/SandboxIR/RegionTest.cpp
@@ -292,3 +292,99 @@ define void @foo(i8 %v0, i8 %v1, i8 %v2) {
EXPECT_EQ(SB.getBeforeCost(), GetCost(LLVMAdd2));
EXPECT_EQ(SB.getAfterCost(), GetCost(LLVMAdd1));
}
+
+TEST_F(RegionTest, Aux) {
+ parseIR(C, R"IR(
+define void @foo(i8 %v) {
+ %t0 = add i8 %v, 0, !sandboxvec !0, !sbaux !2
+ %t1 = add i8 %v, 1, !sandboxvec !0, !sbaux !3
+ %t2 = add i8 %v, 2, !sandboxvec !1
+ %t3 = add i8 %v, 3, !sandboxvec !1, !sbaux !2
+ %t4 = add i8 %v, 4, !sandboxvec !1, !sbaux !4
+ %t5 = add i8 %v, 5, !sandboxvec !1, !sbaux !3
+ ret void
+}
+
+!0 = distinct !{!"sandboxregion"}
+!1 = distinct !{!"sandboxregion"}
+
+!2 = !{i32 0}
+!3 = !{i32 1}
+!4 = !{i32 2}
+)IR");
+ llvm::Function *LLVMF = &*M->getFunction("foo");
+ sandboxir::Context Ctx(C);
+ auto *F = Ctx.createFunction(LLVMF);
+ auto *BB = &*F->begin();
+ auto It = BB->begin();
+ auto *T0 = cast<sandboxir::Instruction>(&*It++);
+ auto *T1 = cast<sandboxir::Instruction>(&*It++);
+ auto *T2 = cast<sandboxir::Instruction>(&*It++);
+ auto *T3 = cast<sandboxir::Instruction>(&*It++);
+ auto *T4 = cast<sandboxir::Instruction>(&*It++);
+ auto *T5 = cast<sandboxir::Instruction>(&*It++);
+
+ SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
+ sandboxir::Region::createRegionsFromMD(*F, *TTI);
+ // Check that the regions are correct.
+ EXPECT_THAT(Regions[0]->insts(), testing::UnorderedElementsAre(T0, T1));
+ EXPECT_THAT(Regions[1]->insts(),
+ testing::UnorderedElementsAre(T2, T3, T4, T5));
+ // Check aux.
+ EXPECT_THAT(Regions[0]->getAux(), testing::ElementsAre(T0, T1));
+ EXPECT_THAT(Regions[1]->getAux(), testing::ElementsAre(T3, T5, T4));
+}
+
+// Check that Aux is well-formed.
+TEST_F(RegionTest, AuxVerify) {
+ parseIR(C, R"IR(
+define void @foo(i8 %v) {
+ %t0 = add i8 %v, 0, !sandboxvec !0, !sbaux !2
+ %t1 = add i8 %v, 1, !sandboxvec !0, !sbaux !3
+ ret void
+}
+
+!0 = distinct !{!"sandboxregion"}
+!2 = !{i32 0}
+!3 = !{i32 2}
+)IR");
+ llvm::Function *LLVMF = &*M->getFunction("foo");
+ sandboxir::Context Ctx(C);
+ auto *F = Ctx.createFunction(LLVMF);
+#ifndef NDEBUG
+ EXPECT_DEATH(sandboxir::Region::createRegionsFromMD(*F, *TTI), ".*Gap*");
+#endif
+}
+
+TEST_F(RegionTest, AuxRoundTrip) {
+ parseIR(C, R"IR(
+define i8 @foo(i8 %v0, i8 %v1) {
+ %t0 = add i8 %v0, 1
+ %t1 = add i8 %t0, %v1
+ ret i8 %t1
+}
+)IR");
+ llvm::Function *LLVMF = &*M->getFunction("foo");
+ sandboxir::Context Ctx(C);
+ auto *F = Ctx.createFunction(LLVMF);
+ auto *BB = &*F->begin();
+ auto It = BB->begin();
+ auto *T0 = cast<sandboxir::Instruction>(&*It++);
+ auto *T1 = cast<sandboxir::Instruction>(&*It++);
+
+ sandboxir::Region Rgn(Ctx, *TTI);
+ Rgn.add(T0);
+ Rgn.add(T1);
+#ifndef NDEBUG
+ EXPECT_DEATH(Rgn.setAux({T0, T0}), ".*already.*");
+#endif
+ Rgn.setAux({T1, T0});
+
+ SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
+ sandboxir::Region::createRegionsFromMD(*F, *TTI);
+ ASSERT_EQ(1U, Regions.size());
+#ifndef NDEBUG
+ EXPECT_EQ(Rgn, *Regions[0].get());
+#endif
+ EXPECT_THAT(Rgn.getAux(), testing::ElementsAre(T1, T0));
+}
>From cc5235beb17a01a4e221c6247bed14e20442efbc Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas at google.com>
Date: Wed, 29 Jan 2025 09:53:39 -0800
Subject: [PATCH 3/4] [SandboxVec] Move seed selection into its own separate
pass
This patch moves the seed selection logic from the BottomUpVec pass into
a new Sandbox IR Function pass. The new "seed-selection" pass collects the
seeds, builds a region and runs the region pass pipeline.
---
.../SandboxVectorizer/Passes/BottomUpVec.h | 14 +--
.../SandboxVectorizer/Passes/SeedSelection.h | 36 +++++++
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 1 +
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 97 +++----------------
.../SandboxVectorizer/Passes/PassRegistry.def | 3 +-
.../Passes/SeedSelection.cpp | 97 +++++++++++++++++++
.../SandboxVectorizer/SandboxVectorizer.cpp | 6 +-
.../SandboxVectorizerPassBuilder.cpp | 1 +
.../SandboxVectorizer/bottomup_basic.ll | 2 +-
.../SandboxVectorizer/bottomup_seed_slice.ll | 2 +-
.../bottomup_seed_slice_pow2.ll | 4 +-
.../Transforms/SandboxVectorizer/cross_bbs.ll | 2 +-
.../default_pass_pipeline.ll | 3 +-
.../test/Transforms/SandboxVectorizer/pack.ll | 2 +-
.../SandboxVectorizer/user_pass_pipeline.ll | 8 +-
15 files changed, 167 insertions(+), 111 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h
create mode 100644 llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index 147a86de4e34ec2..4712ea7eae29ee6 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -16,14 +16,13 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/SandboxIR/Constant.h"
#include "llvm/SandboxIR/Pass.h"
-#include "llvm/SandboxIR/PassManager.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
namespace llvm::sandboxir {
-class BottomUpVec final : public FunctionPass {
+class BottomUpVec final : public RegionPass {
bool Change = false;
std::unique_ptr<LegalityAnalysis> Legality;
/// The original instructions that are potentially dead after vectorization.
@@ -55,16 +54,9 @@ class BottomUpVec final : public FunctionPass {
/// Entry point for vectorization starting from \p Seeds.
bool tryVectorize(ArrayRef<Value *> Seeds);
- /// The PM containing the pipeline of region passes.
- RegionPassManager RPM;
-
public:
- BottomUpVec(StringRef Pipeline);
- bool runOnFunction(Function &F, const Analyses &A) final;
- void printPipeline(raw_ostream &OS) const final {
- OS << getName() << "\n";
- RPM.printPipeline(OS);
- }
+ BottomUpVec() : RegionPass("bottom-up-vec") {}
+ bool runOnRegion(Region &Rgn, const Analyses &A) final;
};
} // namespace llvm::sandboxir
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h
new file mode 100644
index 000000000000000..f93e3d6677fa5de
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h
@@ -0,0 +1,36 @@
+//===- SeedSelection.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The seed-selection pass of the bottom-up vectorizer
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDSELECTION_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDSELECTION_H
+
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/PassManager.h"
+
+namespace llvm::sandboxir {
+
+class SeedSelection final : public FunctionPass {
+
+ /// The PM containing the pipeline of region passes.
+ RegionPassManager RPM;
+
+public:
+ SeedSelection(StringRef Pipeline);
+ bool runOnFunction(Function &F, const Analyses &A) final;
+ void printPipeline(raw_ostream &OS) const final {
+ OS << getName() << "\n";
+ RPM.printPipeline(OS);
+ }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDSELECTION_H
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 872e055294d5574..88e10402e5f6559 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/Legality.cpp
SandboxVectorizer/Passes/BottomUpVec.cpp
SandboxVectorizer/Passes/RegionsFromMetadata.cpp
+ SandboxVectorizer/Passes/SeedSelection.cpp
SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
SandboxVectorizer/SandboxVectorizer.cpp
SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 507d16324012700..b53eeebe117f2f1 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -14,20 +14,10 @@
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
#include "llvm/SandboxIR/Utils.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
namespace llvm {
-static cl::opt<unsigned>
- OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden,
- cl::desc("Override the vector register size in bits, "
- "which is otherwise found by querying TTI."));
-static cl::opt<bool>
- AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
- cl::desc("Allow non-power-of-2 vectorization."));
-
#ifndef NDEBUG
static cl::opt<bool>
AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
@@ -37,10 +27,6 @@ static cl::opt<bool>
namespace sandboxir {
-BottomUpVec::BottomUpVec(StringRef Pipeline)
- : FunctionPass("bottom-up-vec"),
- RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {}
-
static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
unsigned OpIdx) {
SmallVector<Value *, 4> Operands;
@@ -388,6 +374,7 @@ Value *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
}
bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) {
+ Change = false;
DeadInstrCandidates.clear();
Legality->clear();
vectorizeRec(Bndl, {}, /*Depth=*/0);
@@ -395,83 +382,21 @@ bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) {
return Change;
}
-bool BottomUpVec::runOnFunction(Function &F, const Analyses &A) {
+bool BottomUpVec::runOnRegion(Region &Rgn, const Analyses &A) {
+ const auto &SeedSlice = Rgn.getAux();
+ assert(SeedSlice.size() >= 2 && "Bad slice!");
+ Function &F = *SeedSlice[0]->getParent()->getParent();
IMaps = std::make_unique<InstrMaps>(F.getContext());
Legality = std::make_unique<LegalityAnalysis>(
A.getAA(), A.getScalarEvolution(), F.getParent()->getDataLayout(),
F.getContext(), *IMaps);
- Change = false;
- const auto &DL = F.getParent()->getDataLayout();
- unsigned VecRegBits =
- OverrideVecRegBits != 0
- ? OverrideVecRegBits
- : A.getTTI()
- .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
- .getFixedValue();
-
- // TODO: Start from innermost BBs first
- for (auto &BB : F) {
- SeedCollector SC(&BB, A.getScalarEvolution());
- for (SeedBundle &Seeds : SC.getStoreSeeds()) {
- unsigned ElmBits =
- Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType(
- Seeds[Seeds.getFirstUnusedElementIdx()])),
- DL);
-
- auto DivideBy2 = [](unsigned Num) {
- auto Floor = VecUtils::getFloorPowerOf2(Num);
- if (Floor == Num)
- return Floor / 2;
- return Floor;
- };
- // Try to create the largest vector supported by the target. If it fails
- // reduce the vector size by half.
- for (unsigned SliceElms = std::min(VecRegBits / ElmBits,
- Seeds.getNumUnusedBits() / ElmBits);
- SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
- if (Seeds.allUsed())
- break;
- // Keep trying offsets after FirstUnusedElementIdx, until we vectorize
- // the slice. This could be quite expensive, so we enforce a limit.
- for (unsigned Offset = Seeds.getFirstUnusedElementIdx(),
- OE = Seeds.size();
- Offset + 1 < OE; Offset += 1) {
- // Seeds are getting used as we vectorize, so skip them.
- if (Seeds.isUsed(Offset))
- continue;
- if (Seeds.allUsed())
- break;
- auto SeedSlice =
- Seeds.getSlice(Offset, SliceElms * ElmBits, !AllowNonPow2);
- if (SeedSlice.empty())
- continue;
-
- assert(SeedSlice.size() >= 2 && "Should have been rejected!");
-
- // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
- SmallVector<Value *> SeedSliceVals(SeedSlice.begin(),
- SeedSlice.end());
- // Create an empty region. Instructions get added to the region
- // automatically by the callbacks.
- auto &Ctx = F.getContext();
- Region Rgn(Ctx, A.getTTI());
- // Save the state of the IR before we make any changes. The
- // transaction gets accepted/reverted by the tr-accept-or-revert pass.
- Ctx.save();
- // Try to vectorize starting from the seed slice. The returned value
- // is true if we found vectorizable code and generated some vector
- // code for it. It does not mean that the code is profitable.
- bool VecSuccess = tryVectorize(SeedSliceVals);
- if (VecSuccess)
- // WARNING: All passes should return false, except those that
- // accept/revert the state.
- Change |= RPM.runOnRegion(Rgn, A);
- }
- }
- }
- }
- return Change;
+ // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
+ SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
+ // Try to vectorize starting from the seed slice. The returned value
+ // is true if we found vectorizable code and generated some vector
+ // code for it. It does not mean that the code is profitable.
+ return tryVectorize(SeedSliceVals);
}
} // namespace sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index f3aa12729860ff6..384cd98e6a016cd 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -21,6 +21,7 @@ REGION_PASS("null", ::llvm::sandboxir::NullPass)
REGION_PASS("print-instruction-count", ::llvm::sandboxir::PrintInstructionCount)
REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
+REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
#undef REGION_PASS
@@ -28,7 +29,7 @@ REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS_NAME)
#endif
-FUNCTION_PASS_WITH_PARAMS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
+FUNCTION_PASS_WITH_PARAMS("seed-selection", ::llvm::sandboxir::SeedSelection)
FUNCTION_PASS_WITH_PARAMS("regions-from-metadata", ::llvm::sandboxir::RegionsFromMetadata)
#undef FUNCTION_PASS_WITH_PARAMS
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp
new file mode 100644
index 000000000000000..fd2854f0426a383
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp
@@ -0,0 +1,97 @@
+//===- SeedSelection.cpp - Seed selection pass of the bottom-up vectorizer ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/SandboxIR/Module.h"
+#include "llvm/SandboxIR/Region.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+namespace llvm {
+
+static cl::opt<unsigned>
+ OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden,
+ cl::desc("Override the vector register size in bits, "
+ "which is otherwise found by querying TTI."));
+static cl::opt<bool>
+ AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
+ cl::desc("Allow non-power-of-2 vectorization."));
+
+namespace sandboxir {
+SeedSelection::SeedSelection(StringRef Pipeline)
+ : FunctionPass("seed-selection"),
+ RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {}
+
+bool SeedSelection::runOnFunction(Function &F, const Analyses &A) {
+ bool Change = false;
+ const auto &DL = F.getParent()->getDataLayout();
+ unsigned VecRegBits =
+ OverrideVecRegBits != 0
+ ? OverrideVecRegBits
+ : A.getTTI()
+ .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+ .getFixedValue();
+
+ // TODO: Start from innermost BBs first
+ for (auto &BB : F) {
+ SeedCollector SC(&BB, A.getScalarEvolution());
+ for (SeedBundle &Seeds : SC.getStoreSeeds()) {
+ unsigned ElmBits =
+ Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType(
+ Seeds[Seeds.getFirstUnusedElementIdx()])),
+ DL);
+
+ auto DivideBy2 = [](unsigned Num) {
+ auto Floor = VecUtils::getFloorPowerOf2(Num);
+ if (Floor == Num)
+ return Floor / 2;
+ return Floor;
+ };
+ // Try to create the largest vector supported by the target. If it fails
+ // reduce the vector size by half.
+ for (unsigned SliceElms = std::min(VecRegBits / ElmBits,
+ Seeds.getNumUnusedBits() / ElmBits);
+ SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
+ if (Seeds.allUsed())
+ break;
+ // Keep trying offsets after FirstUnusedElementIdx, until we vectorize
+ // the slice. This could be quite expensive, so we enforce a limit.
+ for (unsigned Offset = Seeds.getFirstUnusedElementIdx(),
+ OE = Seeds.size();
+ Offset + 1 < OE; Offset += 1) {
+ // Seeds are getting used as we vectorize, so skip them.
+ if (Seeds.isUsed(Offset))
+ continue;
+ if (Seeds.allUsed())
+ break;
+
+ auto SeedSlice =
+ Seeds.getSlice(Offset, SliceElms * ElmBits, !AllowNonPow2);
+ if (SeedSlice.empty())
+ continue;
+
+ assert(SeedSlice.size() >= 2 && "Should have been rejected!");
+
+ // Create a region containing the seed slice.
+ auto &Ctx = F.getContext();
+ Region Rgn(Ctx, A.getTTI());
+ // TODO: Replace save() with a save pass in the pass pipeline.
+ Ctx.save();
+ Rgn.setAux(SeedSlice);
+ // Run the region pass pipeline.
+ Change |= RPM.runOnRegion(Rgn, A);
+ }
+ }
+ }
+ }
+ return Change;
+}
+} // namespace sandboxir
+} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
index b233d35212f9471..cade41b00f4e319 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
@@ -32,9 +32,11 @@ static cl::opt<std::string> UserDefinedPassPipeline(
SandboxVectorizerPass::SandboxVectorizerPass() : FPM("fpm") {
if (UserDefinedPassPipeline == DefaultPipelineMagicStr) {
// TODO: Add passes to the default pipeline. It currently contains:
- // - the bottom-up-vectorizer pass
+ // - Seed selection, which creates seed regions and runs the pipeline
+ // - Bottom-up Vectorizer pass that starts from a seed
+ // - Accept or revert IR state pass
FPM.setPassPipeline(
- "bottom-up-vec<tr-accept-or-revert>",
+ "seed-selection<bottom-up-vec,tr-accept-or-revert>",
sandboxir::SandboxVectorizerPassBuilder::createFunctionPass);
} else {
// Create the user-defined pipeline.
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 0c1ab55e91a5cfc..f3d044cad62fd8a 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -4,6 +4,7 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
index ee8592c04b62c0a..009f359b93e9f83 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @store_load(ptr %ptr) {
; CHECK-LABEL: define void @store_load(
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
index 202b5a6fbd6c968..219cc7b73027a22 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
declare void @foo()
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
index f1c6e3297d79c7f..c7c3ff9a1b946f7 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
define void @pow2(ptr %ptr, float %val) {
; POW2-LABEL: define void @pow2(
diff --git a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
index ff1604173c31754..9a2ea4d6134350b 100644
--- a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @cross_bbs(ptr %ptr) {
; CHECK-LABEL: define void @cross_bbs(
diff --git a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
index 10de4338caf2325..f0740d41fc508da 100644
--- a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
@@ -5,8 +5,9 @@
; This checks the default pass pipeline for the sandbox vectorizer.
define void @pipeline() {
; CHECK: fpm
-; CHECK: bottom-up-vec
+; CHECK: seed-selection
; CHECK: rpm
+; CHECK: bottom-up-vec
; CHECK: tr-accept-or-revert
; CHECK-EMPTY:
ret void
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index da41036e3a58b76..3843bd5d8161ed6 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
diff --git a/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll
index b11b55ed9601935..4a24b85725dbc4c 100644
--- a/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll
@@ -1,9 +1,9 @@
; RUN: opt -passes=sandbox-vectorizer -sbvec-print-pass-pipeline \
-; RUN: -disable-output -sbvec-passes="bottom-up-vec<null,null>" %s \
+; RUN: -disable-output -sbvec-passes="seed-selection<null,null>" %s \
; RUN: | FileCheck %s
;
; RUN: opt -passes=sandbox-vectorizer -sbvec-print-pass-pipeline \
-; RUN: -disable-output -sbvec-passes="bottom-up-vec<>,regions-from-metadata<>" %s \
+; RUN: -disable-output -sbvec-passes="seed-selection<>,regions-from-metadata<>" %s \
; RUN: | FileCheck --check-prefix CHECK-MULTIPLE-FUNCTION-PASSES %s
; !!!WARNING!!! This won't get updated by update_test_checks.py !
@@ -14,14 +14,14 @@ define void @pipeline() {
}
; CHECK: fpm
-; CHECK: bottom-up-vec
+; CHECK: seed-selection
; CHECK: rpm
; CHECK: null
; CHECK: null
; CHECK-EMPTY:
; CHECK-MULTIPLE-FUNCTION-PASSES: fpm
-; CHECK-MULTIPLE-FUNCTION-PASSES: bottom-up-vec
+; CHECK-MULTIPLE-FUNCTION-PASSES: seed-selection
; CHECK-MULTIPLE-FUNCTION-PASSES: rpm
; CHECK-MULTIPLE-FUNCTION-PASSES: regions-from-metadata
; CHECK-MULTIPLE-FUNCTION-PASSES: rpm
>From d3a4e2e71b8fba78588c82571b091058a0cd8e7e Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas at google.com>
Date: Wed, 29 Jan 2025 10:24:51 -0800
Subject: [PATCH 4/4] [SandboxVec] Replace hard-coded context save() with
transaction-save pass
This patch implements a small region pass that saves the context's state.
The patch is now used in the default pipeline to save the context state
instead of the hard-coded call to Context::save().
The concept behind this is that the passes themselves should not have to
do the actual saving/restoring of the IR state, because that would make it
challenging to reorder them in the pipeline. Having separate save/restore
passes makes the transformation passes more composable as parts of arbitrary
pipelines.
---
.../Passes/TransactionSave.h | 28 +++++++++++++++++++
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 1 +
.../SandboxVectorizer/Passes/PassRegistry.def | 1 +
.../Passes/SeedSelection.cpp | 4 +--
.../Passes/TransactionSave.cpp | 20 +++++++++++++
.../SandboxVectorizer/SandboxVectorizer.cpp | 2 +-
.../SandboxVectorizerPassBuilder.cpp | 1 +
.../SandboxVectorizer/bottomup_basic.ll | 2 +-
.../SandboxVectorizer/bottomup_seed_slice.ll | 2 +-
.../bottomup_seed_slice_pow2.ll | 4 +--
.../Transforms/SandboxVectorizer/cross_bbs.ll | 2 +-
.../test/Transforms/SandboxVectorizer/pack.ll | 2 +-
12 files changed, 59 insertions(+), 10 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.h
create mode 100644 llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.cpp
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.h
new file mode 100644
index 000000000000000..73aafabfd0b0cb9
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.h
@@ -0,0 +1,28 @@
+//===- TransactionSave.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a region pass that simply calls Context::save() to save the IR state.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONSAVE_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONSAVE_H
+
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/Region.h"
+
+namespace llvm::sandboxir {
+
+class TransactionSave : public RegionPass {
+public:
+ TransactionSave() : RegionPass("tr-save") {}
+ bool runOnRegion(Region &Rgn, const Analyses &A) final;
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_TRANSACTIONSAVE_H
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 88e10402e5f6559..d658fb454fd8162 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -11,6 +11,7 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/Passes/RegionsFromMetadata.cpp
SandboxVectorizer/Passes/SeedSelection.cpp
SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
+ SandboxVectorizer/Passes/TransactionSave.cpp
SandboxVectorizer/SandboxVectorizer.cpp
SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
SandboxVectorizer/Scheduler.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index 384cd98e6a016cd..f9cca82b533f6aa 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -19,6 +19,7 @@
REGION_PASS("null", ::llvm::sandboxir::NullPass)
REGION_PASS("print-instruction-count", ::llvm::sandboxir::PrintInstructionCount)
+REGION_PASS("tr-save", ::llvm::sandboxir::TransactionSave)
REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp
index fd2854f0426a383..75ccfc57a6fae59 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.cpp
@@ -6,10 +6,10 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
@@ -82,8 +82,6 @@ bool SeedSelection::runOnFunction(Function &F, const Analyses &A) {
// Create a region containing the seed slice.
auto &Ctx = F.getContext();
Region Rgn(Ctx, A.getTTI());
- // TODO: Replace save() with a save pass in the pass pipeline.
- Ctx.save();
Rgn.setAux(SeedSlice);
// Run the region pass pipeline.
Change |= RPM.runOnRegion(Rgn, A);
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.cpp
new file mode 100644
index 000000000000000..8d39d971273b455
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.cpp
@@ -0,0 +1,20 @@
+//===- TransactionSave.cpp - Save the IR state ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InstructionCost.h"
+
+namespace llvm::sandboxir {
+
+bool TransactionSave::runOnRegion(Region &Rgn, const Analyses &A) {
+ Rgn.getContext().save();
+ return false;
+}
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
index cade41b00f4e319..6eb3d5652e1f4c7 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
@@ -36,7 +36,7 @@ SandboxVectorizerPass::SandboxVectorizerPass() : FPM("fpm") {
// - Bottom-up Vectorizer pass that starts from a seed
// - Accept or revert IR state pass
FPM.setPassPipeline(
- "seed-selection<bottom-up-vec,tr-accept-or-revert>",
+ "seed-selection<tr-save,bottom-up-vec,tr-accept-or-revert>",
sandboxir::SandboxVectorizerPassBuilder::createFunctionPass);
} else {
// Create the user-defined pipeline.
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index f3d044cad62fd8a..f74a95c745ce1d7 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -7,6 +7,7 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedSelection.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionSave.h"
namespace llvm::sandboxir {
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
index 009f359b93e9f83..a317beb698b50e4 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @store_load(ptr %ptr) {
; CHECK-LABEL: define void @store_load(
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
index 219cc7b73027a22..aea4be9912947b1 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
declare void @foo()
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
index c7c3ff9a1b946f7..8cd2f77fe215f57 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="seed-selection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="seed-selection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
define void @pow2(ptr %ptr, float %val) {
; POW2-LABEL: define void @pow2(
diff --git a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
index 9a2ea4d6134350b..61f7b8f99be1e67 100644
--- a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @cross_bbs(ptr %ptr) {
; CHECK-LABEL: define void @cross_bbs(
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index 3843bd5d8161ed6..07126019316c647 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-selection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
More information about the llvm-commits
mailing list