[llvm] fa92d51 - [VP] Merge ExpandVP pass into PreISelIntrinsicLowering (#101652)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 6 00:28:03 PDT 2024
Author: Alexis Engelke
Date: 2024-08-06T09:27:59+02:00
New Revision: fa92d51f9e609270bf14c4bc16eb42a98183d27f
URL: https://github.com/llvm/llvm-project/commit/fa92d51f9e609270bf14c4bc16eb42a98183d27f
DIFF: https://github.com/llvm/llvm-project/commit/fa92d51f9e609270bf14c4bc16eb42a98183d27f.diff
LOG: [VP] Merge ExpandVP pass into PreISelIntrinsicLowering (#101652)
Similar to #97727; avoid an extra pass over the entire IR by performing
the lowering as part of the pre-isel-intrinsic-lowering pass.
Added:
llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-fp-intrinsics.ll
llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-gather-scatter.ll
llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-load-store.ll
llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp.ll
Modified:
llvm/include/llvm/CodeGen/ExpandVectorPredication.h
llvm/include/llvm/CodeGen/Passes.h
llvm/include/llvm/LinkAllPasses.h
llvm/include/llvm/Passes/MachinePassRegistry.def
llvm/lib/CodeGen/ExpandVectorPredication.cpp
llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
llvm/lib/CodeGen/TargetPassConfig.cpp
llvm/test/CodeGen/AArch64/O0-pipeline.ll
llvm/test/CodeGen/AArch64/O3-pipeline.ll
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
llvm/test/CodeGen/ARM/O3-pipeline.ll
llvm/test/CodeGen/LoongArch/O0-pipeline.ll
llvm/test/CodeGen/LoongArch/opt-pipeline.ll
llvm/test/CodeGen/PowerPC/O0-pipeline.ll
llvm/test/CodeGen/PowerPC/O3-pipeline.ll
llvm/test/CodeGen/RISCV/O0-pipeline.ll
llvm/test/CodeGen/RISCV/O3-pipeline.ll
llvm/test/CodeGen/X86/O0-pipeline.ll
llvm/test/CodeGen/X86/opt-pipeline.ll
llvm/tools/llc/llc.cpp
llvm/tools/opt/optdriver.cpp
Removed:
llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll
llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll
llvm/test/CodeGen/Generic/expand-vp-load-store.ll
llvm/test/CodeGen/Generic/expand-vp.ll
################################################################################
diff --git a/llvm/include/llvm/CodeGen/ExpandVectorPredication.h b/llvm/include/llvm/CodeGen/ExpandVectorPredication.h
index b69adb9db2601..c42c644c99e91 100644
--- a/llvm/include/llvm/CodeGen/ExpandVectorPredication.h
+++ b/llvm/include/llvm/CodeGen/ExpandVectorPredication.h
@@ -13,11 +13,14 @@
namespace llvm {
-class ExpandVectorPredicationPass
- : public PassInfoMixin<ExpandVectorPredicationPass> {
-public:
- PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-};
+class TargetTransformInfo;
+class VPIntrinsic;
+
+/// Expand a vector predication intrinsic. Returns true if the intrinsic was
+/// removed/replaced.
+bool expandVectorPredicationIntrinsic(VPIntrinsic &VPI,
+ const TargetTransformInfo &TTI);
+
} // end namespace llvm
#endif // LLVM_CODEGEN_EXPANDVECTORPREDICATION_H
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index cafb9781698a2..20273d069bf05 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -513,11 +513,6 @@ namespace llvm {
// the corresponding function in a vector library (e.g., SVML, libmvec).
FunctionPass *createReplaceWithVeclibLegacyPass();
- /// This pass expands the vector predication intrinsics into unpredicated
- /// instructions with selects or just the explicit vector length into the
- /// predicate mask.
- FunctionPass *createExpandVectorPredicationPass();
-
// Expands large div/rem instructions.
FunctionPass *createExpandLargeDivRemPass();
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index d23d59862abba..c00e425b13198 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -119,7 +119,6 @@ namespace {
(void) llvm::createMergeICmpsLegacyPass();
(void) llvm::createExpandLargeDivRemPass();
(void)llvm::createExpandMemCmpLegacyPass();
- (void) llvm::createExpandVectorPredicationPass();
std::string buf;
llvm::raw_string_ostream os(buf);
(void) llvm::createPrintModulePass(os);
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index e702721c299a8..8e669ee579123 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -53,7 +53,6 @@ FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(TM))
FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass(TM))
FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass(TM))
FUNCTION_PASS("expand-reductions", ExpandReductionsPass())
-FUNCTION_PASS("expandvp", ExpandVectorPredicationPass())
FUNCTION_PASS("gc-lowering", GCLoweringPass())
FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass(TM))
FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM))
diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
index 97c6ee4773f22..5ffdbcda93e75 100644
--- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp
+++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
//
-// This pass implements IR expansion for vector predication intrinsics, allowing
+// This file implements IR expansion for vector predication intrinsics, allowing
// targets to enable vector predication until just before codegen.
//
//===----------------------------------------------------------------------===//
@@ -16,7 +16,6 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
-#include "llvm/CodeGen/Passes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
@@ -24,8 +23,6 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
@@ -137,7 +134,6 @@ namespace {
// Expansion pass state at function scope.
struct CachingVPExpander {
- Function &F;
const TargetTransformInfo &TTI;
/// \returns A (fixed length) vector with ascending integer indices
@@ -207,10 +203,10 @@ struct CachingVPExpander {
bool UsingTTIOverrides;
public:
- CachingVPExpander(Function &F, const TargetTransformInfo &TTI)
- : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
+ CachingVPExpander(const TargetTransformInfo &TTI)
+ : TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
- bool expandVectorPredication();
+ bool expandVectorPredication(VPIntrinsic &VPI);
};
//// CachingVPExpander {
@@ -571,7 +567,7 @@ CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
VPIntrinsic &VPI) {
assert(VPI.canIgnoreVectorLengthParam());
- const auto &DL = F.getDataLayout();
+ const auto &DL = VPI.getDataLayout();
Value *MaskParam = VPI.getMaskParam();
Value *PtrParam = VPI.getMemoryPointerParam();
@@ -775,15 +771,6 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
//// } CachingVPExpander
-struct TransformJob {
- VPIntrinsic *PI;
- TargetTransformInfo::VPLegalization Strategy;
- TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat)
- : PI(PI), Strategy(InitStrat) {}
-
- bool isDone() const { return Strategy.shouldDoNothing(); }
-};
-
void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) {
// Operations with speculatable lanes do not strictly need predication.
if (maySpeculateLanes(VPI)) {
@@ -821,98 +808,43 @@ CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
}
/// Expand llvm.vp.* intrinsics as requested by \p TTI.
-bool CachingVPExpander::expandVectorPredication() {
- SmallVector<TransformJob, 16> Worklist;
-
- // Collect all VPIntrinsics that need expansion and determine their expansion
- // strategy.
- for (auto &I : instructions(F)) {
- auto *VPI = dyn_cast<VPIntrinsic>(&I);
- if (!VPI)
- continue;
- auto VPStrat = getVPLegalizationStrategy(*VPI);
- sanitizeStrategy(*VPI, VPStrat);
- if (!VPStrat.shouldDoNothing())
- Worklist.emplace_back(VPI, VPStrat);
- }
- if (Worklist.empty())
- return false;
+bool CachingVPExpander::expandVectorPredication(VPIntrinsic &VPI) {
+ auto Strategy = getVPLegalizationStrategy(VPI);
+ sanitizeStrategy(VPI, Strategy);
- // Transform all VPIntrinsics on the worklist.
- LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size()
- << " instructions ::::\n");
- for (TransformJob Job : Worklist) {
- // Transform the EVL parameter.
- switch (Job.Strategy.EVLParamStrategy) {
- case VPLegalization::Legal:
- break;
- case VPLegalization::Discard:
- discardEVLParameter(*Job.PI);
- break;
- case VPLegalization::Convert:
- if (foldEVLIntoMask(*Job.PI))
- ++NumFoldedVL;
- break;
- }
- Job.Strategy.EVLParamStrategy = VPLegalization::Legal;
+ // Transform the EVL parameter.
+ switch (Strategy.EVLParamStrategy) {
+ case VPLegalization::Legal:
+ break;
+ case VPLegalization::Discard:
+ discardEVLParameter(VPI);
+ break;
+ case VPLegalization::Convert:
+ if (foldEVLIntoMask(VPI))
+ ++NumFoldedVL;
+ break;
+ }
- // Replace with a non-predicated operation.
- switch (Job.Strategy.OpStrategy) {
- case VPLegalization::Legal:
- break;
- case VPLegalization::Discard:
- llvm_unreachable("Invalid strategy for operators.");
- case VPLegalization::Convert:
- expandPredication(*Job.PI);
+ // Replace with a non-predicated operation.
+ switch (Strategy.OpStrategy) {
+ case VPLegalization::Legal:
+ break;
+ case VPLegalization::Discard:
+ llvm_unreachable("Invalid strategy for operators.");
+ case VPLegalization::Convert:
+ if (Value *V = expandPredication(VPI); V != &VPI) {
++NumLoweredVPOps;
- break;
+ // Return true if and only if the intrinsic was actually removed.
+ return true;
}
- Job.Strategy.OpStrategy = VPLegalization::Legal;
-
- assert(Job.isDone() && "incomplete transformation");
+ break;
}
- return true;
+ return false;
}
-class ExpandVectorPredication : public FunctionPass {
-public:
- static char ID;
- ExpandVectorPredication() : FunctionPass(ID) {
- initializeExpandVectorPredicationPass(*PassRegistry::getPassRegistry());
- }
-
- bool runOnFunction(Function &F) override {
- const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
- CachingVPExpander VPExpander(F, *TTI);
- return VPExpander.expandVectorPredication();
- }
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<TargetTransformInfoWrapperPass>();
- AU.setPreservesCFG();
- }
-};
} // namespace
-char ExpandVectorPredication::ID;
-INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp",
- "Expand vector predication intrinsics", false, false)
-INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp",
- "Expand vector predication intrinsics", false, false)
-
-FunctionPass *llvm::createExpandVectorPredicationPass() {
- return new ExpandVectorPredication();
-}
-
-PreservedAnalyses
-ExpandVectorPredicationPass::run(Function &F, FunctionAnalysisManager &AM) {
- const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
- CachingVPExpander VPExpander(F, TTI);
- if (!VPExpander.expandVectorPredication())
- return PreservedAnalyses::all();
- PreservedAnalyses PA;
- PA.preserveSet<CFGAnalyses>();
- return PA;
+bool llvm::expandVectorPredicationIntrinsic(VPIntrinsic &VPI,
+ const TargetTransformInfo &TTI) {
+ return CachingVPExpander(TTI).expandVectorPredication(VPI);
}
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 51dad439f57d5..c0e08063be41d 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -16,6 +16,7 @@
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/ExpandVectorPredication.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -351,6 +352,16 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
return Changed;
});
break;
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
+ case Intrinsic::VPID:
+#include "llvm/IR/VPIntrinsics.def"
+ Changed |= forEachCall(F, [&](CallInst *CI) {
+ Function *Parent = CI->getParent()->getParent();
+ const TargetTransformInfo &TTI = LookupTTI(*Parent);
+ auto *VPI = cast<VPIntrinsic>(CI);
+ return expandVectorPredicationIntrinsic(*VPI, TTI);
+ });
+ break;
case Intrinsic::objc_autorelease:
Changed |= lowerObjCCall(F, "objc_autorelease");
break;
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index eb74c6f3e5d44..3a20b340d45f2 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -865,11 +865,6 @@ void TargetPassConfig::addIRPasses() {
if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining)
addPass(createPartiallyInlineLibCallsPass());
- // Expand vector predication intrinsics into standard IR instructions.
- // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction
- // passes since it emits those kinds of intrinsics.
- addPass(createExpandVectorPredicationPass());
-
// Instrument function entry after all inlining.
addPass(createPostInlineEntryExitInstrumenterPass());
diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll
index bfcb9a710f70e..ba611493e1a76 100644
--- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll
@@ -22,7 +22,6 @@
; CHECK-NEXT: Lower Garbage Collection Instructions
; CHECK-NEXT: Shadow Stack GC Lowering
; CHECK-NEXT: Remove unreachable blocks from the CFG
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index 017349aa32af3..845634e8e9830 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -60,7 +60,6 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 15f23eda241b4..ee7f9375bf5df 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -43,7 +43,6 @@
; GCN-O0-NEXT: FunctionPass Manager
; GCN-O0-NEXT: Expand Atomic instructions
; GCN-O0-NEXT: Remove unreachable blocks from the CFG
-; GCN-O0-NEXT: Expand vector predication intrinsics
; GCN-O0-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; GCN-O0-NEXT: Scalarize Masked Memory Intrinsics
; GCN-O0-NEXT: Expand reduction intrinsics
@@ -222,7 +221,6 @@
; GCN-O1-NEXT: Constant Hoisting
; GCN-O1-NEXT: Replace intrinsics with calls to vector library
; GCN-O1-NEXT: Partially inline calls to library functions
-; GCN-O1-NEXT: Expand vector predication intrinsics
; GCN-O1-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; GCN-O1-NEXT: Scalarize Masked Memory Intrinsics
; GCN-O1-NEXT: Expand reduction intrinsics
@@ -508,7 +506,6 @@
; GCN-O1-OPTS-NEXT: Constant Hoisting
; GCN-O1-OPTS-NEXT: Replace intrinsics with calls to vector library
; GCN-O1-OPTS-NEXT: Partially inline calls to library functions
-; GCN-O1-OPTS-NEXT: Expand vector predication intrinsics
; GCN-O1-OPTS-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; GCN-O1-OPTS-NEXT: Scalarize Masked Memory Intrinsics
; GCN-O1-OPTS-NEXT: Expand reduction intrinsics
@@ -813,7 +810,6 @@
; GCN-O2-NEXT: Constant Hoisting
; GCN-O2-NEXT: Replace intrinsics with calls to vector library
; GCN-O2-NEXT: Partially inline calls to library functions
-; GCN-O2-NEXT: Expand vector predication intrinsics
; GCN-O2-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; GCN-O2-NEXT: Scalarize Masked Memory Intrinsics
; GCN-O2-NEXT: Expand reduction intrinsics
@@ -1126,7 +1122,6 @@
; GCN-O3-NEXT: Constant Hoisting
; GCN-O3-NEXT: Replace intrinsics with calls to vector library
; GCN-O3-NEXT: Partially inline calls to library functions
-; GCN-O3-NEXT: Expand vector predication intrinsics
; GCN-O3-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; GCN-O3-NEXT: Scalarize Masked Memory Intrinsics
; GCN-O3-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll
index aa92c7ab4fb9f..e74e4f2ad3893 100644
--- a/llvm/test/CodeGen/ARM/O3-pipeline.ll
+++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll
@@ -38,7 +38,6 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
index 138f0c81238b3..38c1dbcb1075f 100644
--- a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
@@ -26,7 +26,6 @@
; CHECK-NEXT: Lower Garbage Collection Instructions
; CHECK-NEXT: Shadow Stack GC Lowering
; CHECK-NEXT: Remove unreachable blocks from the CFG
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
index c5c5342e303c7..391888a38daf6 100644
--- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
@@ -61,7 +61,6 @@
; LAXX-NEXT: Constant Hoisting
; LAXX-NEXT: Replace intrinsics with calls to vector library
; LAXX-NEXT: Partially inline calls to library functions
-; LAXX-NEXT: Expand vector predication intrinsics
; LAXX-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; LAXX-NEXT: Scalarize Masked Memory Intrinsics
; LAXX-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/PowerPC/O0-pipeline.ll b/llvm/test/CodeGen/PowerPC/O0-pipeline.ll
index f1c9da078a16e..70b421f8c0c5f 100644
--- a/llvm/test/CodeGen/PowerPC/O0-pipeline.ll
+++ b/llvm/test/CodeGen/PowerPC/O0-pipeline.ll
@@ -25,7 +25,6 @@
; CHECK-NEXT: Lower Garbage Collection Instructions
; CHECK-NEXT: Shadow Stack GC Lowering
; CHECK-NEXT: Remove unreachable blocks from the CFG
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll
index be0fbf3abcda2..f4f492782eb65 100644
--- a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll
+++ b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll
@@ -62,7 +62,6 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/RISCV/O0-pipeline.ll b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
index fd2ba49ea8610..7b2eec894a44d 100644
--- a/llvm/test/CodeGen/RISCV/O0-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
@@ -26,7 +26,6 @@
; CHECK-NEXT: Lower Garbage Collection Instructions
; CHECK-NEXT: Shadow Stack GC Lowering
; CHECK-NEXT: Remove unreachable blocks from the CFG
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index d6d0cca6ddae7..df3c690af155b 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -62,7 +62,6 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index 29d3c2795ffb4..98b86384b8443 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -26,7 +26,6 @@
; CHECK-NEXT: Lower Garbage Collection Instructions
; CHECK-NEXT: Shadow Stack GC Lowering
; CHECK-NEXT: Remove unreachable blocks from the CFG
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
index 275a42e02d746..6fcc1ed068f43 100644
--- a/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -59,7 +59,6 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
-; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
diff --git a/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-fp-intrinsics.ll
similarity index 99%
rename from llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll
rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-fp-intrinsics.ll
index bc89ddea6b85a..d0b6fe27d8233 100644
--- a/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-fp-intrinsics.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -expandvp -S < %s | FileCheck %s
+; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s
define void @vp_fadd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind {
; CHECK-LABEL: define void @vp_fadd_v4f32(
diff --git a/llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-gather-scatter.ll
similarity index 99%
rename from llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll
rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-gather-scatter.ll
index 2e2dba50c8459..23096eccc6136 100644
--- a/llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-gather-scatter.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt --expandvp -S < %s | FileCheck %s
+; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s
; Fixed vectors
define <4 x i32> @vpgather_v4i32(<4 x ptr> %ptrs, <4 x i1> %m, i32 zeroext %evl) {
diff --git a/llvm/test/CodeGen/Generic/expand-vp-load-store.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-load-store.ll
similarity index 97%
rename from llvm/test/CodeGen/Generic/expand-vp-load-store.ll
rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-load-store.ll
index 5c6f1e858ce7c..c966c3e7de0a2 100644
--- a/llvm/test/CodeGen/Generic/expand-vp-load-store.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-load-store.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt --expandvp -S < %s | FileCheck %s
-; RUN: opt --expandvp --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s
+; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s
+; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s
; Fixed vectors
define <2 x i64> @vpload_v2i64(ptr %ptr, <2 x i1> %m, i32 zeroext %evl) {
diff --git a/llvm/test/CodeGen/Generic/expand-vp.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp.ll
similarity index 97%
rename from llvm/test/CodeGen/Generic/expand-vp.ll
rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp.ll
index 4fee9a533b947..92b36054356b2 100644
--- a/llvm/test/CodeGen/Generic/expand-vp.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp.ll
@@ -1,12 +1,12 @@
; Partial expansion cases (still VP with parameter expansions).
-; RUN: opt --expandvp --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=LEGAL_LEGAL
-; RUN: opt --expandvp --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=DISCARD_LEGAL
-; RUN: opt --expandvp --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=CONVERT_LEGAL
+; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=LEGAL_LEGAL
+; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=DISCARD_LEGAL
+; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=CONVERT_LEGAL
; Full expansion cases (all expanded to non-VP).
-; RUN: opt --expandvp --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
-; RUN: opt --expandvp -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
-; RUN: opt --expandvp --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
-; RUN: opt --expandvp --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
+; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
+; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
+; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
+; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT
; Fixed-width vectors
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index e7bf192192b67..d3f7c2b7f819b 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -342,7 +342,6 @@ int main(int argc, char **argv) {
initializeVectorization(*Registry);
initializeScalarizeMaskedMemIntrinLegacyPassPass(*Registry);
initializeExpandReductionsPass(*Registry);
- initializeExpandVectorPredicationPass(*Registry);
initializeHardwareLoopsLegacyPass(*Registry);
initializeTransformUtils(*Registry);
initializeReplaceWithVeclibLegacyPass(*Registry);
diff --git a/llvm/tools/opt/optdriver.cpp b/llvm/tools/opt/optdriver.cpp
index b26aeb061851e..eb4821d521405 100644
--- a/llvm/tools/opt/optdriver.cpp
+++ b/llvm/tools/opt/optdriver.cpp
@@ -443,7 +443,6 @@ extern "C" int optMain(
initializePostInlineEntryExitInstrumenterPass(Registry);
initializeUnreachableBlockElimLegacyPassPass(Registry);
initializeExpandReductionsPass(Registry);
- initializeExpandVectorPredicationPass(Registry);
initializeWasmEHPreparePass(Registry);
initializeWriteBitcodePassPass(Registry);
initializeReplaceWithVeclibLegacyPass(Registry);
More information about the llvm-commits
mailing list