[llvm] [VPlan] Track VPValue names in VPlan. (PR #81411)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 12 08:51:41 PDT 2024
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/81411
>From 5c96c8d51863fb6c2b96e06ffcd7b90280bb20ca Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 10 Feb 2024 21:49:22 +0000
Subject: [PATCH 1/8] [VPlan] Track VPValue names in VPlan.
This patch restructures the way names for printing VPValues are handled.
It updates VPlan to track assigned names for VPValues (similar to how
LLVMContext tracks names for LLVM IR Values). Tracking names in a single
place allows to deduplicate names and ensure all names are unique.
Now there are the following cases:
* If a VPValue has a defining recipe, its name is determined by
checking its associated VPlan's mapping. The name is wrapped in vp<>.
If no VPlan is associated, <badref> is used.
* If a VPValue is a live-in with an underlying IR value, the name of
the IR value is used and wrapped in ir<>.
* If none of the above applies, the number assigned by VPSlotTracker is
used wrapped in vp<> if an associated VPlan is provided or <badref>
otherwise.
Note that contrary to LLMV's Value which always can retrieve
LLVMContext, this is not possible for VPValues without defining recipe,
so in some cases, the associated VPlan needs to be provided explicitly.
The new handling subsumes VPInstruction's manual handling of its name.
---
.../Vectorize/LoopVectorizationPlanner.h | 23 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 13 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 82 +++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 66 ++++-
.../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 9 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 16 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 20 +-
.../AArch64/sve-tail-folding-forced.ll | 14 +-
.../LoopVectorize/AArch64/sve-widen-gep.ll | 20 +-
.../AArch64/synthesize-mask-for-call.ll | 84 +++---
.../widen-call-with-intrinsic-or-libfunc.ll | 28 +-
.../RISCV/riscv-vector-reverse.ll | 36 +--
.../first-order-recurrence-chains-vplan.ll | 46 ++--
...-order-recurrence-sink-replicate-region.ll | 132 ++++-----
.../Transforms/LoopVectorize/icmp-uniforms.ll | 10 +-
.../interleave-and-scalarize-only.ll | 20 +-
.../LoopVectorize/vplan-dot-printing.ll | 14 +-
.../LoopVectorize/vplan-iv-transforms.ll | 6 +-
.../vplan-printing-outer-loop.ll | 26 +-
.../LoopVectorize/vplan-printing.ll | 252 +++++++++---------
.../vplan-sink-scalars-and-merge-vf1.ll | 8 +-
.../vplan-sink-scalars-and-merge.ll | 250 ++++++++---------
.../vplan-unused-interleave-group.ll | 6 +-
24 files changed, 662 insertions(+), 531 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index a7ebf78e54ceb6..5aa4892bf0ffe5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -47,16 +47,19 @@ class VPBuilder {
VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
/// Insert \p VPI in BB at InsertPt if BB is set.
- VPInstruction *tryInsertInstruction(VPInstruction *VPI) {
- if (BB)
+ VPInstruction *tryInsertInstruction(VPInstruction *VPI,
+ const Twine &Name = "") {
+ if (BB) {
BB->insert(VPI, InsertPt);
+ VPI->setName(Name);
+ }
return VPI;
}
VPInstruction *createInstruction(unsigned Opcode,
ArrayRef<VPValue *> Operands, DebugLoc DL,
const Twine &Name = "") {
- return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
+ return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL), Name);
}
VPInstruction *createInstruction(unsigned Opcode,
@@ -150,7 +153,7 @@ class VPBuilder {
VPRecipeWithIRFlags::WrapFlagsTy WrapFlags,
DebugLoc DL = {}, const Twine &Name = "") {
return tryInsertInstruction(
- new VPInstruction(Opcode, Operands, WrapFlags, DL, Name));
+ new VPInstruction(Opcode, Operands, WrapFlags, DL), Name);
}
VPValue *createNot(VPValue *Operand, DebugLoc DL = {},
const Twine &Name = "") {
@@ -170,12 +173,12 @@ class VPBuilder {
VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
DebugLoc DL = {}, const Twine &Name = "",
std::optional<FastMathFlags> FMFs = std::nullopt) {
- auto *Select =
- FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
- *FMFs, DL, Name)
- : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
- DL, Name);
- return tryInsertInstruction(Select);
+ auto *Select = FMFs
+ ? new VPInstruction(Instruction::Select,
+ {Cond, TrueVal, FalseVal}, *FMFs, DL)
+ : new VPInstruction(Instruction::Select,
+ {Cond, TrueVal, FalseVal}, DL);
+ return tryInsertInstruction(Select, Name);
}
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1a7b301c35f2b8..d08aa8fcb12c24 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7262,7 +7262,7 @@ VPValue *VPBuilder::createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
return tryInsertInstruction(
- new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name));
+ new VPInstruction(Instruction::ICmp, Pred, A, B, DL), Name);
}
// This function will select a scalable VF if the target supports scalable
@@ -8610,11 +8610,11 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
// IV by VF * UF.
auto *CanonicalIVIncrement =
new VPInstruction(Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()},
- {HasNUW, false}, DL, "index.next");
+ {HasNUW, false}, DL);
CanonicalIVPHI->addOperand(CanonicalIVIncrement);
VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
- EB->appendRecipe(CanonicalIVIncrement);
+ EB->appendRecipe(CanonicalIVIncrement, "index.next");
// Add the BranchOnCount VPInstruction to the latch.
VPInstruction *BranchBack =
@@ -8781,8 +8781,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
CM.foldTailByMasking() || isa<TruncInst>(Instr)) &&
"unexpected recipe needs moving");
Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
+ Recipe->getVPSingleValue()->setName(Instr->getName());
} else
- VPBB->appendRecipe(Recipe);
+ VPBB->appendRecipe(Recipe, Instr->getName());
}
VPBlockUtils::insertBlockAfter(new VPBasicBlock(), VPBB);
@@ -8842,6 +8843,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
J++;
}
MemberR->eraseFromParent();
+ if (!Member->getType()->isVoidTy())
+ VPIG->getVPValue(J - 1)->setName(Member->getName());
}
}
@@ -9097,7 +9100,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
// ensure that it comes after all of it's inputs, including CondOp.
// Note that this transformation may leave over dead recipes (including
// CurrentLink), which will be cleaned by a later VPlan transform.
- LinkVPBB->appendRecipe(RedRecipe);
+ LinkVPBB->appendRecipe(RedRecipe, CurrentLinkI->getName());
CurrentLink->replaceAllUsesWith(RedRecipe);
PreviousLink = RedRecipe;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2c0daa82afa59f..bb603653f37ddc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -588,6 +588,15 @@ bool VPBasicBlock::isExiting() const {
return getParent()->getExitingBasicBlock() == this;
}
+void VPBasicBlock::appendRecipe(VPRecipeBase *Recipe, const Twine &Name) {
+ insert(Recipe, end());
+ if (Recipe->getNumDefinedValues() == 1)
+ Recipe->getVPSingleValue()->setName(Name);
+ else
+ assert(Name.str().empty() && "Provided a name for a recipe that does not "
+ "define exactly a single VPValue");
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const {
if (getSuccessors().empty()) {
@@ -1053,8 +1062,10 @@ static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry,
assert(OldR.getNumDefinedValues() == NewR.getNumDefinedValues() &&
"recipes must define the same number of operands");
for (const auto &[OldV, NewV] :
- zip(OldR.definedValues(), NewR.definedValues()))
+ zip(OldR.definedValues(), NewR.definedValues())) {
Old2NewVPValues[OldV] = NewV;
+ NewV->setName(OldV->getName());
+ }
}
}
@@ -1112,6 +1123,26 @@ VPlan *VPlan::duplicate() {
return NewPlan;
}
+void VPlan::setName(const VPValue *V, const Twine &Name) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ std::string N = Name.str();
+ if (N.empty())
+ return;
+ assert(!VPValue2Name.contains(V));
+ std::string CurrName = N;
+
+ if (UsedNames.contains(N))
+ CurrName = N + ".1";
+ unsigned Cnt = 2;
+ while (UsedNames.contains(CurrName)) {
+ CurrName = N + "." + std::to_string(Cnt);
+ Cnt += 1;
+ }
+ VPValue2Name[V] = CurrName;
+ UsedNames.insert(CurrName);
+#endif
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1298,18 +1329,12 @@ void VPValue::replaceUsesWithIf(
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const {
- if (const Value *UV = getUnderlyingValue()) {
+ if (!getDefiningRecipe() && getUnderlyingValue()) {
OS << "ir<";
- UV->printAsOperand(OS, false);
+ getUnderlyingValue()->printAsOperand(OS, false);
OS << ">";
- return;
- }
-
- unsigned Slot = Tracker.getSlot(this);
- if (Slot == unsigned(-1))
- OS << "<badref>";
- else
- OS << "vp<%" << Tracker.getSlot(this) << ">";
+ } else
+ OS << Tracker.getName(this);
}
void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
@@ -1319,6 +1344,31 @@ void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
}
#endif
+std::string VPValue::getName() {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ if (auto *Def = getDefiningRecipe())
+ return Def->getParent()->getPlan()->getName(this);
+ assert(isLiveIn() && "called getName() on unsupported VPValue");
+ return getLiveInIRValue()->getName().str();
+#else
+ return "";
+#endif
+}
+
+void VPValue::setName(const Twine &Name) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ auto *Def = getDefiningRecipe();
+ Def->getParent()->getPlan()->setName(this, Name);
+#endif
+}
+
+void VPValue::takeName(const VPValue *Src) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ auto *Def = getDefiningRecipe();
+ Def->getParent()->getPlan()->takeName(Src, this);
+#endif
+}
+
void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
Old2NewTy &Old2New,
InterleavedAccessInfo &IAI) {
@@ -1397,6 +1447,16 @@ void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
assignSlot(Def);
}
+std::string VPSlotTracker::getName(const VPValue *V) const {
+ if (!Plan)
+ return "<badref>";
+ std::string N = Plan->getName(V);
+ if (!N.empty())
+ return (Twine("vp<%") + N + ">").str();
+
+ return (Twine("vp<%") + std::to_string(getSlot(V)) + ">").str();
+}
+
bool vputils::onlyFirstLaneUsed(const VPValue *Def) {
return all_of(Def->users(),
[Def](const VPUser *U) { return U->onlyFirstLaneUsed(Def); });
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 162a3c4b195e53..08c0041f23ae07 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -32,6 +32,7 @@
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/ilist.h"
#include "llvm/ADT/ilist_node.h"
@@ -1162,9 +1163,6 @@ class VPInstruction : public VPRecipeWithIRFlags {
typedef unsigned char OpcodeTy;
OpcodeTy Opcode;
- /// An optional name that can be used for the generated IR instruction.
- const std::string Name;
-
/// Utility method serving execute(): generates a single instance of the
/// modeled instruction. \returns the generated value for \p Part.
/// In some cases an existing value is returned rather than a generated
@@ -1181,31 +1179,30 @@ class VPInstruction : public VPRecipeWithIRFlags {
void setUnderlyingInstr(Instruction *I) { setUnderlyingValue(I); }
public:
- VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
- const Twine &Name = "")
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
- Opcode(Opcode), Name(Name.str()) {}
+ Opcode(Opcode) {}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- DebugLoc DL = {}, const Twine &Name = "")
- : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}
+ DebugLoc DL = {})
+ : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL) {}
VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
- VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
+ VPValue *B, DebugLoc DL = {});
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "")
+ WrapFlagsTy WrapFlags, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL),
- Opcode(Opcode), Name(Name.str()) {}
+ Opcode(Opcode) {}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
+ FastMathFlags FMFs, DebugLoc DL = {});
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
VPRecipeBase *clone() override {
SmallVector<VPValue *, 2> Operands(operands());
- auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name);
+ auto *New = new VPInstruction(Opcode, Operands, getDebugLoc());
New->transferFlags(*this);
return New;
}
@@ -2653,7 +2650,7 @@ class VPBasicBlock : public VPBlockBase {
/// Augment the existing recipes of a VPBasicBlock with an additional
/// \p Recipe as the last recipe.
- void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
+ void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = "");
/// The method which generates the output IR instructions that correspond to
/// this VPBasicBlock, thereby "executing" the VPlan.
@@ -2876,6 +2873,11 @@ class VPlan {
/// been modeled in VPlan directly.
DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ DenseMap<const VPValue *, std::string> VPValue2Name;
+ StringSet<> UsedNames;
+#endif
+
public:
/// Construct a VPlan with original preheader \p Preheader, trip count \p TC
/// and \p Entry to the plan. At the moment, \p Preheader and \p Entry need to
@@ -3016,6 +3018,42 @@ class VPlan {
LLVM_DUMP_METHOD void dump() const;
#endif
+ /// Set the name for \p V to \p Name if it is not empty. \p V must not have a
+ /// name assigned already.
+ void setName(const VPValue *V, const Twine &Name);
+
+ /// Remove the assigned name for \p V, if there is one.
+ void removeName(const VPValue *V) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ if (VPValue2Name.contains(V)) {
+ UsedNames.erase(VPValue2Name[V]);
+ VPValue2Name.erase(V);
+ }
+#endif
+ }
+
+ /// Take the name for \p Src and move it to \p Dst. The name of \p Src will be
+ /// empty afterwards.
+ void takeName(const VPValue *Src, const VPValue *Dst) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ std::string N = VPValue2Name.lookup(Src);
+ if (N.empty())
+ return;
+ VPValue2Name[Dst] = N;
+ VPValue2Name.erase(Src);
+#endif
+ }
+
+ /// Return the name assigned to \p V or an empty string if no name has been
+ /// assigned.
+ std::string getName(const VPValue *V) const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ return VPValue2Name.lookup(V);
+#else
+ return "";
+#endif
+ }
+
/// Returns a range mapping the values the range \p Operands to their
/// corresponding VPValues.
iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 94456bf858d9c5..f5b34a740a21b0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -297,7 +297,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
if (Br->isConditional()) {
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
VPBB->appendRecipe(
- new VPInstruction(VPInstruction::BranchOnCond, {Cond}));
+ new VPInstruction(VPInstruction::BranchOnCond, {Cond}),
+ Inst->getName());
}
// Skip the rest of the Instruction processing for Branch instructions.
@@ -310,7 +311,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// an empty VPInstruction that we will fix once the whole plain CFG has
// been built.
NewVPV = new VPWidenPHIRecipe(Phi);
- VPBB->appendRecipe(cast<VPWidenPHIRecipe>(NewVPV));
+ VPBB->appendRecipe(cast<VPWidenPHIRecipe>(NewVPV), Inst->getName());
PhisToFix.push_back(Phi);
} else {
// Translate LLVM-IR operands into VPValue operands and set them in the
@@ -321,8 +322,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// Build VPInstruction for any arbitrary Instruction without specific
// representation in VPlan.
- NewVPV = cast<VPInstruction>(
- VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
+ NewVPV = cast<VPInstruction>(VPIRBuilder.createNaryOp(
+ Inst->getOpcode(), VPOperands, Inst, Inst->getName()));
}
IRDef2VPValue[Inst] = NewVPV;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9ee0cb2bd61530..c24e0d72c222f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -226,6 +226,8 @@ void VPRecipeBase::removeFromParent() {
iplist<VPRecipeBase>::iterator VPRecipeBase::eraseFromParent() {
assert(getParent() && "Recipe not in any VPBasicBlock");
+ for (auto *V : definedValues())
+ getParent()->getPlan()->removeName(V);
return getParent()->getRecipeList().erase(getIterator());
}
@@ -255,20 +257,19 @@ FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const {
}
VPInstruction::VPInstruction(unsigned Opcode, CmpInst::Predicate Pred,
- VPValue *A, VPValue *B, DebugLoc DL,
- const Twine &Name)
+ VPValue *A, VPValue *B, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef<VPValue *>({A, B}),
Pred, DL),
- Opcode(Opcode), Name(Name.str()) {
+ Opcode(Opcode) {
assert(Opcode == Instruction::ICmp &&
"only ICmp predicates supported at the moment");
}
VPInstruction::VPInstruction(unsigned Opcode,
std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL, const Twine &Name)
+ FastMathFlags FMFs, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL),
- Opcode(Opcode), Name(Name.str()) {
+ Opcode(Opcode) {
// Make sure the VPInstruction is a floating-point operation.
assert(isFPMathOp() && "this op can't take fast-math flags");
}
@@ -277,6 +278,7 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
unsigned Part) {
IRBuilderBase &Builder = State.Builder;
Builder.SetCurrentDebugLocation(getDebugLoc());
+ std::string Name = getParent()->getPlan()->getName(this);
if (Instruction::isBinaryOp(getOpcode())) {
if (Part != 0 && vputils::onlyFirstPartUsed(this))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 71f5285f90236b..bb1db219704439 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -88,9 +88,10 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
}
NewRecipe->insertBefore(&Ingredient);
- if (NewRecipe->getNumDefinedValues() == 1)
+ if (NewRecipe->getNumDefinedValues() == 1) {
VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
- else
+ NewRecipe->getVPSingleValue()->takeName(VPV);
+ } else
assert(NewRecipe->getNumDefinedValues() == 0 &&
"Only recpies with zero or one defined values expected");
Ingredient.eraseFromParent();
@@ -163,6 +164,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
// TODO: add ".cloned" suffix to name of Clone's VPValue.
Clone->insertBefore(SinkCandidate);
+ if (!I->getName().empty())
+ Clone->setName(I->getName() + ".cloned");
SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) {
return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
});
@@ -318,6 +321,8 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
PHIRecipe->setOperand(0, RecipeWithoutMask);
}
PredRecipe->eraseFromParent();
+ Plan.setName(RecipeWithoutMask,
+ RecipeWithoutMask->getUnderlyingValue()->getName());
auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true);
@@ -1177,10 +1182,11 @@ void VPlanTransforms::addActiveLaneMask(
LaneMask = addVPLaneMaskPhiAndUpdateExitBranch(
Plan, DataAndControlFlowWithoutRuntimeCheck);
} else {
- LaneMask = new VPInstruction(VPInstruction::ActiveLaneMask,
- {WideCanonicalIV, Plan.getTripCount()},
- nullptr, "active.lane.mask");
+ LaneMask =
+ new VPInstruction(VPInstruction::ActiveLaneMask,
+ {WideCanonicalIV, Plan.getTripCount()}, nullptr);
LaneMask->insertAfter(WideCanonicalIV);
+ LaneMask->setName("active.lane.mask");
}
// Walk users of WideCanonicalIV and replace all compares of the form
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index c85f7715feaa2a..e7ddc472484481 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -192,6 +192,21 @@ class VPValue {
/// is a live-in value.
/// TODO: Also handle recipes defined in pre-header blocks.
bool isDefinedOutsideVectorRegions() const { return !hasDefiningRecipe(); }
+
+ /// Returns the name associated with the VPValue. In case the VPValue has a
+ /// defining recipe, look up the name in the containing VPlan. In case it is a
+ /// live-in with an underlying IR value, return the name of the underlying
+ /// value. Calling getName() in other cases is invalid.
+ std::string getName();
+
+ /// Sets the name for this VPValue to \p Name. The VPValue must have a
+ /// defining recipe inserted in a VPlan.
+ void setName(const Twine &Name);
+
+ /// Take the name for \p Src and move it to this VPValue. The name of \p Src
+ /// will be empty afterwards. The VPValue must have a defining recipe inserted
+ /// in a VPlan.
+ void takeName(const VPValue *Src);
};
typedef DenseMap<Value *, VPValue *> Value2VPValueTy;
@@ -446,6 +461,7 @@ class VPBasicBlock;
/// VPlan and allows querying the numbering for printing, similar to the
/// ModuleSlotTracker for IR values.
class VPSlotTracker {
+ const VPlan *Plan;
DenseMap<const VPValue *, unsigned> Slots;
unsigned NextSlot = 0;
@@ -454,7 +470,7 @@ class VPSlotTracker {
void assignSlots(const VPBasicBlock *VPBB);
public:
- VPSlotTracker(const VPlan *Plan = nullptr) {
+ VPSlotTracker(const VPlan *Plan = nullptr) : Plan(Plan) {
if (Plan)
assignSlots(*Plan);
}
@@ -465,6 +481,8 @@ class VPSlotTracker {
return -1;
return I->second;
}
+
+ std::string getName(const VPValue *V) const;
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
index cc72dfa4ce639f..33708bad471da3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
@@ -19,20 +19,20 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]>
-; VPLANS-NEXT: EMIT vp<[[VF:%[0-9]+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%[0-9]+]]> = active lane mask vp<[[VF]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
+; VPLANS-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, vp<[[TC]]>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
; VPLANS-NEXT: <x1> vector loop: {
; VPLANS-NEXT: vector.body:
; VPLANS-NEXT: EMIT vp<[[INDV:%[0-9]+]]> = CANONICAL-INDUCTION
-; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%[0-9]+]]>
+; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<[[LANEMASK_LOOP:%.+]]>
; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1>
-; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
-; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer ir<%gep>
+; VPLANS-NEXT: CLONE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
+; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer vp<%gep>
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
-; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
-; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]>
+; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
+; VPLANS-NEXT: EMIT vp<[[INC:%.+]]> = VF * Part + vp<[[INDV]]>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[NEWTC]]>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index 899fcce5c02aba..eb7e5997b96c87 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -19,16 +19,16 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: EMIT ir<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8
-; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
-; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ptr.iv.1>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR2]]>
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1>
-; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer ir<%ptr.iv.2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, ir<%add>
+; CHECK-NEXT: EMIT vp<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8
+; CHECK-NEXT: EMIT vp<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
+; CHECK-NEXT: WIDEN-GEP Var[Inv] vp<%ptr.iv.2.next> = getelementptr inbounds vp<%ptr.iv.2>, ir<1>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ptr.iv.1>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%ptr.iv.2.next>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%ptr.iv.2>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR2]]>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<%lv>, ir<1>
+; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer vp<%ptr.iv.2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index ad0d40ee7fbde9..02e348a8e6f57b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -21,13 +21,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REPLICATE vp<%call> = call @foo(vp<%load>)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -50,13 +50,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -84,13 +84,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -113,13 +113,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir<true>) (using library function: foo_vector_fixed4_mask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>, ir<true>) (using library function: foo_vector_fixed4_mask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -146,13 +146,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -175,13 +175,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
index f67b932b04a0d6..9b8cbb37ea2de7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
@@ -19,13 +19,13 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double
-; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64)
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst>
+; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double
+; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using library function: __simd_sin_v2f64)
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -48,13 +48,13 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double
-; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic)
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst>
+; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double
+; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using vector intrinsic)
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 1bcd7a2e009e0b..ec70c0e3657fd0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -64,15 +64,15 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
+; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0>
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%add9> = add vp<[[L]]>, ir<1>
+; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add9>
; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -204,15 +204,15 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
-; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1>
+; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0>
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%conv1> = fadd vp<[[L]]>, ir<1.000000e+00>
+; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%conv1>
; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index 15e77f3a48470c..db85e9d8a34c9a 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -15,17 +15,17 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next>
-; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]>
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
+; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next>
+; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -68,20 +68,20 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next>
-; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]>
-; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice ir<%for.3>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add.2>
+; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next>
+; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]>
+; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice vp<%for.3>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: WIDEN vp<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: WIDEN vp<%add.2> = add vp<%add.1>, vp<[[FOR3_SPLICE]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add.2>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 833d55f09294e8..06f7a7be352f4b 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -20,10 +20,10 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -32,19 +32,19 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED1]]> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED1]]> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -53,14 +53,14 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%conv>, vp<%rem>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -111,11 +111,11 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -124,15 +124,15 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%rem>, vp<%recur.next>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
@@ -181,12 +181,12 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%and.red> = phi ir<1234>, vp<%and.red.next>
; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
; CHECK-NEXT: Successor(s): pred.srem
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.srem: {
@@ -195,19 +195,19 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
; CHECK-NEXT: Successor(s): pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next>
-; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add>
-; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, vp<%recur.next>
+; CHECK-NEXT: WIDEN vp<%and.red.next> = and vp<%and.red>, vp<%add>
+; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, vp<%and.red.next>, vp<%and.red>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -215,7 +215,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%and.red>, vp<[[SEL]]>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]>
@@ -257,11 +257,11 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -270,18 +270,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
+; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED]]> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED]]> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK: <xVFxUF> pred.store: {
@@ -290,18 +290,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep>
-; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2>
-; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%lv.2> = load vp<%gep>
+; CHECK-NEXT: REPLICATE vp<%conv.lv.2> = sext vp<%lv.2>
+; CHECK-NEXT: REPLICATE vp<%add.1> = add vp<%conv>, vp<%rem>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%add.1>, vp<%conv.lv.2>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%rem>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.2>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
@@ -360,11 +360,11 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -374,17 +374,17 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep>
-; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%rem.div> = sdiv ir<20>, vp<%rem>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep.2>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem.div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem.div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
@@ -435,12 +435,12 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%.pn> = phi ir<0>, vp<[[L:%.+]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]>, vp<[[BTC]]>
-; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src>
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn>, ir<[[L]]>
+; CHECK-NEXT: CLONE vp<[[L]]> = load ir<%src>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%.pn>, vp<[[L]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -449,14 +449,14 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%val> = sdiv vp<[[SPLICE]]>, ir<%x>
; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%val>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = vp<%val>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index b4ab6f7e8ceb70..f0b8ded142210d 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -49,9 +49,9 @@ for.end:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13>
-; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>
+; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%cond0> = icmp ult vp<%iv>, ir<13>
+; CHECK-NEXT: WIDEN-SELECT vp<%s> = select vp<%cond0>, ir<10>, ir<20>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -61,8 +61,8 @@ for.end:
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index 81cc2024bb31a5..54a66dd424faed 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -21,9 +21,9 @@
; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%start> + vp<[[CAN_IV]]> * ir<1>
; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>)
-; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]>
-; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx>
+; DBG-NEXT: CLONE vp<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>)
+; DBG-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]>
+; DBG-NEXT: CLONE store vp<%min>, vp<%arrayidx>
; DBG-NEXT: EMIT vp<[[INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; DBG-NEXT: EMIT branch-on-count vp<[[INC]]>, vp<[[VEC_TC]]>
; DBG-NEXT: No successors
@@ -89,14 +89,14 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-EMPTY:
; DBG-NEXT: pred.store.if:
; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; DBG-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]>
-; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src>
-; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]>
-; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst>
+; DBG-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]>
+; DBG-NEXT: CLONE vp<%l> = load vp<%gep.src>
+; DBG-NEXT: CLONE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]>
+; DBG-NEXT: CLONE store vp<%l>, vp<%gep.dst>
; DBG-NEXT: Successor(s): pred.store.continue
; DBG-EMPTY:
; DBG-NEXT: pred.store.continue:
-; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l>
+; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = vp<%l>
; DBG-NEXT: No successors
; DBG-NEXT: }
; DBG-NEXT: Successor(s): cond.false.1
@@ -190,10 +190,10 @@ exit:
; DBG-NEXT: <x1> vector loop: {
; DBG-NEXT: vector.body:
; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]>
+; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]>
; DBG-NEXT: SCALAR-CAST vp<[[TRUNC_IV:%.+]]> = trunc vp<[[CAN_IV]]> to i32
; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[TRUNC_IV]]>, vp<[[CAST]]>
-; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[SCALAR_STEPS]]>
+; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[SCALAR_STEPS]]>
; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst>
; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VTC]]>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index 12b8e657aabf12..85d035f1ddaee0 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -28,13 +28,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: "vector.body:\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION ir\<0\>, vp\<[[CAN_IV_NEXT:%.+]]\>\l" +
; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" +
-; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer ir\<%arrayidx\>\l" +
-; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
-; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" +
-; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer ir\<%arrayidx2\>\l" +
-; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" +
+; CHECK-NEXT: " CLONE vp\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer vp\<%arrayidx\>\l" +
+; CHECK-NEXT: " WIDEN vp\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
+; CHECK-NEXT: " WIDEN-CALL vp\<%call\> = call @llvm.sqrt.f32(vp\<%lv\>) (using vector intrinsic)\l" +
+; CHECK-NEXT: " CLONE vp\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer vp\<%arrayidx2\>\l" +
+; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, vp\<%call\>\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" +
; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" +
; CHECK-NEXT: "No successors\l"
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 7ab2459ada2ed6..00996e7eee5664 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -16,9 +16,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next.p, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%iv>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%iv>
; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 65df5973ac75b6..1795c6a4fe33f5 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -15,28 +15,28 @@ define void @foo(i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
-; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
-; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1>
-; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
+; CHECK-NEXT: WIDEN-PHI vp<%outer.iv> = phi ir<0>, vp<%outer.iv.next>
+; CHECK-NEXT: EMIT vp<%gep.1> = getelementptr ir<@arr2>, ir<0>, vp<%outer.iv>
+; CHECK-NEXT: EMIT store vp<%outer.iv>, vp<%gep.1>
+; CHECK-NEXT: EMIT vp<%add> = add vp<%outer.iv>, ir<%n>
; CHECK-NEXT: Successor(s): inner
; CHECK-EMPTY:
; CHECK-NEXT: <x1> inner: {
; CHECK-NEXT: inner:
-; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
-; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
-; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2>
-; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
+; CHECK-NEXT: WIDEN-PHI vp<%inner.iv> = phi ir<0>, vp<%inner.iv.next>
+; CHECK-NEXT: EMIT vp<%gep.2> = getelementptr ir<@arr>, ir<0>, vp<%inner.iv>, vp<%outer.iv>
+; CHECK-NEXT: EMIT store vp<%add>, vp<%gep.2>
+; CHECK-NEXT: EMIT vp<%inner.iv.next> = add vp<%inner.iv>, ir<1>
+; CHECK-NEXT: EMIT vp<%inner.ec> = icmp vp<%inner.iv.next>, ir<8>
+; CHECK-NEXT: EMIT branch-on-cond vp<%inner.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): outer.latch
; CHECK-EMPTY:
; CHECK-NEXT: outer.latch:
-; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
+; CHECK-NEXT: EMIT vp<%outer.iv.next> = add vp<%outer.iv>, ir<1>
+; CHECK-NEXT: EMIT vp<%outer.ec> = icmp vp<%outer.iv.next>, ir<8>
+; CHECK-NEXT: EMIT branch-on-cond vp<%outer.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 89178953010fe0..372e99583c4066 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -20,13 +20,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @llvm.sqrt.f32(vp<%lv>)
+; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -71,15 +71,15 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr inbounds ir<%y>, ir<%iv>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z>
-; CHECK-NEXT: WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
-; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
+; CHECK-NEXT: WIDEN-GEP Inv[Var] vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<%iv>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%cmp> = icmp eq vp<%arrayidx>, ir<%z>
+; CHECK-NEXT: WIDEN-SELECT vp<%sel> = select vp<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
+; CHECK-NEXT: WIDEN vp<%add> = fadd vp<%lv>, vp<%sel>
+; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -124,12 +124,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -137,7 +137,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%red>, ir<%red.next>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out float %red.next.lcssa = vp<[[RED_RES]]>
@@ -173,12 +173,12 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1>
; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>) (with final reduction value stored in invariant address sank outside of loop)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -186,7 +186,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result ir<%red>, ir<%red.next>
+; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
@@ -227,30 +227,30 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5>
+; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%i>, ir<5>
; CHECK-NEXT: Successor(s): pred.udiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.udiv: {
; CHECK-NEXT: pred.udiv.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp>
+; CHECK-NEXT: BRANCH-ON-MASK vp<%cmp>
; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.if:
-; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
+; CHECK-NEXT: REPLICATE vp<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
; CHECK-NEXT: Successor(s): pred.udiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%tmp4>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%tmp4>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%cmp>
-; CHECK-NEXT: BLEND ir<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp>
-; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%d>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%cmp>
+; CHECK-NEXT: BLEND vp<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/vp<%cmp>
+; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%d>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -303,19 +303,19 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4>
-; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0>
-; CHECK-NEXT: ir<%AB.0> = load from index 0
-; CHECK-NEXT: ir<%AB.1> = load from index 1
-; CHECK-NEXT: ir<%AB.3> = load from index 3
-; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<[[STEPS]]>, ir<3>
-; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%AB.0>, ir<%AB.1>
-; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, ir<%iv.plus.3>
-; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%gep.CD.3>
-; CHECK-NEXT: store ir<%add> to index 0
+; CHECK-NEXT: CLONE vp<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, vp<%gep.AB.0>
+; CHECK-NEXT: vp<%AB.0> = load from index 0
+; CHECK-NEXT: vp<%AB.1> = load from index 1
+; CHECK-NEXT: vp<%AB.3> = load from index 3
+; CHECK-NEXT: CLONE vp<%iv.plus.3> = add vp<[[STEPS]]>, ir<3>
+; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%AB.0>, vp<%AB.1>
+; CHECK-NEXT: CLONE vp<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, vp<%iv.plus.3>
+; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at <badref>, vp<%gep.CD.3>
+; CHECK-NEXT: store vp<%add> to index 0
; CHECK-NEXT: store ir<1> to index 1
; CHECK-NEXT: store ir<2> to index 2
-; CHECK-NEXT: store ir<%AB.3> to index 3
+; CHECK-NEXT: store vp<%AB.3> to index 3
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -370,16 +370,16 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%sum.07> = phi ir<0.000000e+00>, vp<%muladd.1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN ir<%l.b> = load vp<[[VEC_PTR2]]>
-; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b>
-; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%l.a> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
+; CHECK-NEXT: WIDEN vp<%l.b> = load vp<[[VEC_PTR2]]>
+; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz vp<%l.a>, vp<%l.b>
+; CHECK-NEXT: REDUCE vp<[[MULADD:%.+]]> = vp<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -387,7 +387,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%sum.07>, ir<[[MULADD]]>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%sum.07>, vp<[[MULADD]]>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out float %muladd.lcssa = vp<[[RED_RES]]>
@@ -426,15 +426,15 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%isd>
-; CHECK-NEXT: WIDEN ir<%lsd> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%psd> = add nuw nsw ir<%lsd>, ir<23>
-; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100>
-; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3
-; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200>
-; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, ir<%cmp2>, ir<false>, !dbg /tmp/s.c:5:21
-; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, ir<%cmp1>
+; CHECK-NEXT: CLONE vp<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%isd>
+; CHECK-NEXT: WIDEN vp<%lsd> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%psd> = add nuw nsw vp<%lsd>, ir<23>
+; CHECK-NEXT: WIDEN vp<%cmp1> = icmp slt vp<%lsd>, ir<100>
+; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not vp<%cmp1>, !dbg /tmp/s.c:5:3
+; CHECK-NEXT: WIDEN vp<%cmp2> = icmp sge vp<%lsd>, ir<200>
+; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, vp<%cmp2>, ir<false>, !dbg /tmp/s.c:5:21
+; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, vp<%cmp1>
; CHECK-NEXT: Successor(s): pred.sdiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.sdiv: {
@@ -443,21 +443,21 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.if:
-; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V)
+; CHECK-NEXT: REPLICATE vp<%sd1> = sdiv vp<%psd>, vp<%lsd> (S->V)
; CHECK-NEXT: Successor(s): pred.sdiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = ir<%sd1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = vp<%sd1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
-; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2>
+; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp2>
; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir<false>
-; CHECK-NEXT: BLEND ir<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%isd>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%ysd.0>
+; CHECK-NEXT: BLEND vp<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> vp<%psd>/vp<[[SEL2]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%isd>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%ysd.0>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -521,12 +521,12 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION\l" +
; CHECK-NEXT: " %iv = phi %iv.next, 0\l" +
-; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]>
+; CHECK-NEXT: " vp<%v2>, vp<[[EXP_SCEV]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]>
-; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep>
+; CHECK-NEXT: WIDEN vp<%v3> = add nuw vp<%v2>, ir<1>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%v3>, vp<%gep>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -572,9 +572,9 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<%iv>, ir<%off>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<0>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
@@ -585,7 +585,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: Live-out i32 %lcssa = ir<%add>
+; CHECK-NEXT: Live-out i32 %lcssa = vp<%add>
; CHECK-NEXT: }
;
entry:
@@ -619,15 +619,15 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.y>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%add> = fadd nnan ir<%lv>, ir<1.000000e+00>
-; CHECK-NEXT: WIDEN ir<%mul> = fmul reassoc nnan ninf nsz arcp contract afn ir<%add>, ir<2.000000e+00>
-; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00>
-; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%div>
+; CHECK-NEXT: CLONE vp<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.y>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%add> = fadd nnan vp<%lv>, ir<1.000000e+00>
+; CHECK-NEXT: WIDEN vp<%mul> = fmul reassoc nnan ninf nsz arcp contract afn vp<%add>, ir<2.000000e+00>
+; CHECK-NEXT: WIDEN vp<%div> = fdiv reassoc nsz contract vp<%mul>, ir<2.000000e+00>
+; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%div>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -672,14 +672,14 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%div.1> = udiv exact ir<%lv>, ir<20>
-; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60>
-; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
+; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%div.1> = udiv exact vp<%lv>, ir<20>
+; CHECK-NEXT: WIDEN vp<%div.2> = udiv vp<%lv>, ir<60>
+; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%div.1>, vp<%div.2>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -723,36 +723,36 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ld.addr>
-; CHECK-NEXT: WIDEN ir<%ld.value> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%ifcond> = fcmp oeq ir<%ld.value>, ir<5.000000e+00>
+; CHECK-NEXT: CLONE vp<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ld.addr>
+; CHECK-NEXT: WIDEN vp<%ld.value> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%ifcond> = fcmp oeq vp<%ld.value>, ir<5.000000e+00>
; CHECK-NEXT: Successor(s): pred.call
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.call: {
; CHECK-NEXT: pred.call.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%ifcond>
+; CHECK-NEXT: BRANCH-ON-MASK vp<%ifcond>
; CHECK-NEXT: Successor(s): pred.call.if, pred.call.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.call.if:
-; CHECK-NEXT: REPLICATE ir<%foo.ret.1> = call nnan ninf nsz @foo(ir<%ld.value>) (S->V)
-; CHECK-NEXT: REPLICATE ir<%foo.ret.2> = call @foo(ir<%ld.value>) (S->V)
+; CHECK-NEXT: REPLICATE vp<%foo.ret.1> = call nnan ninf nsz @foo(vp<%ld.value>) (S->V)
+; CHECK-NEXT: REPLICATE vp<%foo.ret.2> = call @foo(vp<%ld.value>) (S->V)
; CHECK-NEXT: Successor(s): pred.call.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.call.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = ir<%foo.ret.1>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = ir<%foo.ret.2>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = vp<%foo.ret.1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = vp<%foo.ret.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.1
; CHECK-EMPTY:
; CHECK-NEXT: if.then.1:
-; CHECK-NEXT: WIDEN ir<%fadd> = fadd vp<[[PHI1]]>, vp<[[PHI2]]>
-; CHECK-NEXT: EMIT vp<[[NOT_COND:%.+]]> = not ir<%ifcond>
-; CHECK-NEXT: BLEND ir<%st.value> = ir<%ld.value>/vp<[[NOT_COND]]> ir<%fadd>/ir<%ifcond>
-; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%st.addr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%st.value>
+; CHECK-NEXT: WIDEN vp<%fadd> = fadd vp<[[PHI1]]>, vp<[[PHI2]]>
+; CHECK-NEXT: EMIT vp<[[NOT_COND:%.+]]> = not vp<%ifcond>
+; CHECK-NEXT: BLEND vp<%st.value> = vp<%ld.value>/vp<[[NOT_COND]]> vp<%fadd>/vp<%ifcond>
+; CHECK-NEXT: CLONE vp<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%st.addr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%st.value>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -806,14 +806,14 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%or.1> = or disjoint ir<%lv>, ir<1>
-; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3>
-; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%add>
+; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%or.1> = or disjoint vp<%lv>, ir<1>
+; CHECK-NEXT: WIDEN vp<%or.2> = or vp<%lv>, ir<3>
+; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%or.1>, vp<%or.2>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -857,11 +857,11 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l>
-; CHECK-NEXT: REPLICATE store ir<%zext>, ir<%p1>
+; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%p>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST vp<%zext> = zext nneg vp<%l>
+; CHECK-NEXT: REPLICATE store vp<%zext>, ir<%p1>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index 8c18c107b39fc1..7bf5e6bd76bd3d 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -19,9 +19,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2>
-; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2>
+; CHECK-NEXT: CLONE vp<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: CLONE vp<%tmp3> = load vp<%tmp2>
+; CHECK-NEXT: CLONE store ir<0>, vp<%tmp2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -30,7 +30,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
+; CHECK-NEXT: CLONE store vp<%tmp3>, vp<%tmp2>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 9b9c3e704852a1..3019a7e29e4e33 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -27,7 +27,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -37,16 +37,16 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]
-; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add>
-; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%lv.b>, ir<10>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]
+; CHECK-NEXT: REPLICATE vp<%mul> = mul ir<2>, vp<%add>
+; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -96,7 +96,7 @@ exit:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK: <xVFxUF> pred.load: {
@@ -106,17 +106,17 @@ exit:
; CHECK: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: loop.0:
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -125,9 +125,9 @@ exit:
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<[[PRED]]>, ir<10>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
@@ -180,7 +180,7 @@ exit:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK: <xVFxUF> pred.load: {
@@ -190,18 +190,18 @@ exit:
; CHECK: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: loop.0:
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<10>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, ir<10>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, vp<%add>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -210,8 +210,8 @@ exit:
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
@@ -265,10 +265,10 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]>, vp<[[BTC]]>
-; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0>
-; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform>
-; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k>
-; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp>
+; CHECK-NEXT: CLONE vp<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0>
+; CHECK-NEXT: CLONE vp<%lv> = load vp<%gep.A.uniform>
+; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%iv>, ir<%k>
+; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp>
; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<[[NOT2]]>, ir<false>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
@@ -279,8 +279,8 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B>
+; CHECK-NEXT: REPLICATE vp<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%lv>, vp<%gep.B>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -337,10 +337,10 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.1>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%c.1> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.1>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -350,21 +350,21 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.1>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
-; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
+; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -373,8 +373,8 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -437,11 +437,11 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
+; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: WIDEN vp<%c.1> = icmp ugt vp<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -451,22 +451,22 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
-; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
-; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]>, ir<%c.1>, ir<false>
+; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
+; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]>, vp<%c.1>, ir<false>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -475,8 +475,8 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -546,10 +546,10 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
+; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, vp<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -559,22 +559,22 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
-; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
-; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
+; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]>, vp<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -583,10 +583,10 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a>
-; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store ir<0>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.c>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -656,7 +656,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -665,26 +665,26 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
-; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c>
-; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%lv.a>, vp<%gep.c>
+; CHECK-NEXT: REPLICATE store vp<%lv.b>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
; CHECK-EMPTY:
; CHECK-NEXT: loop.3:
-; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, ir<%c.0>, ir<false>
-; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>
+; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<%c.0>, ir<false>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -693,8 +693,8 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1>
+; CHECK-NEXT: REPLICATE vp<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.c.1>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -760,7 +760,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -770,15 +770,15 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
-; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a>
-; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<%lv.a>, vp<%lv.a>
+; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -827,10 +827,10 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[PRED:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -839,17 +839,17 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = vp<%lv.a>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[PRED]]>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[PRED]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -858,12 +858,12 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>
-; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>
+; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -915,15 +915,15 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
-; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
-; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
-; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
-; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>
+; CHECK-NEXT: REPLICATE vp<%l1> = load ir<%src>
+; CHECK-NEXT: REPLICATE vp<%l2> = trunc vp<%l1>
+; CHECK-NEXT: REPLICATE vp<%cmp> = icmp eq vp<%l1>, ir<0>
+; CHECK-NEXT: REPLICATE vp<%sel> = select vp<%cmp>, ir<5>, vp<%l2>
+; CHECK-NEXT: REPLICATE store vp<%sel>, ir<%dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%l1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%l1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.then.1
@@ -973,11 +973,11 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%0> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not ir<%pred>
+; CHECK-NEXT: CLONE vp<%gep.cloned> = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.cloned>
+; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%pred> = fcmp oeq vp<[[L]]>, ir<0.000000e+00>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not vp<%pred>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, vp<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -1055,14 +1055,14 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%l> = load vp<%gep.src>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%l>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = ir<%l>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = vp<%l>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
@@ -1113,12 +1113,12 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1
-; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0>
-; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1>
+; CHECK-NEXT: EMIT vp<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1
+; CHECK-NEXT: CLONE vp<%ptr.iv.next.cloned> = getelementptr inbounds vp<%ptr.iv>, ir<-1>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%ptr.iv.next.cloned>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%c.1> = icmp eq vp<%l>, ir<0>
+; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not vp<%c.1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -1127,8 +1127,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1>
-; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next>
+; CHECK-NEXT: REPLICATE vp<%ptr.iv.next> = getelementptr inbounds vp<%ptr.iv>, ir<-1>
+; CHECK-NEXT: REPLICATE store ir<95>, vp<%ptr.iv.next>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
index 5ea27994b356da..27d81de260d3b9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
@@ -18,9 +18,9 @@ define void @test_unused_interleave(ptr %src, i32 %length) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%3>
-; CHECK-NEXT: EMIT vp<%3> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%3>, vp<%1>
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
>From d72a7b34e175d5ebb78f2300c37618fa2526bfcd Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 11 Mar 2024 09:11:28 +0000
Subject: [PATCH 2/8] Step
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +--
llvm/lib/Transforms/Vectorize/VPlan.cpp | 21 ++++++++++++-------
llvm/lib/Transforms/Vectorize/VPlan.h | 19 ++++++++++-------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 +++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 --
5 files changed, 29 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 850108fd803421..a16615c04730c9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8696,9 +8696,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
CM.foldTailByMasking() || isa<TruncInst>(Instr)) &&
"unexpected recipe needs moving");
Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
- Recipe->getVPSingleValue()->setName(Instr->getName());
} else
- VPBB->appendRecipe(Recipe, Instr->getName());
+ VPBB->appendRecipe(Recipe);
}
VPBlockUtils::insertBlockAfter(new VPBasicBlock(), VPBB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 6413c3f3aad011..459b3b9e3666fc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -606,13 +606,19 @@ bool VPBasicBlock::isExiting() const {
return getParent() && getParent()->getExitingBasicBlock() == this;
}
-void VPBasicBlock::appendRecipe(VPRecipeBase *Recipe, const Twine &Name) {
- insert(Recipe, end());
- if (Recipe->getNumDefinedValues() == 1)
- Recipe->getVPSingleValue()->setName(Name);
- else
- assert(Name.str().empty() && "Provided a name for a recipe that does not "
- "define exactly a single VPValue");
+void VPBasicBlock::insert(VPRecipeBase *Recipe, iterator InsertPt) {
+ assert(Recipe && "No recipe to append.");
+ assert(!Recipe->Parent && "Recipe already in VPlan");
+ Recipe->Parent = this;
+ Recipes.insert(InsertPt, Recipe);
+
+ if (Recipe->getNumDefinedValues() == 1) {
+ VPValue *VPV = Recipe->getVPSingleValue();
+ Value *UV = VPV->getUnderlyingValue();
+ VPlan *ParentPlan = getPlan();
+ if (UV && ParentPlan && !ParentPlan->hasName(VPV))
+ VPV->setName(UV->getName());
+ }
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1140,7 +1146,6 @@ void VPlan::setName(const VPValue *V, const Twine &Name) {
std::string N = Name.str();
if (N.empty())
return;
- assert(!VPValue2Name.contains(V));
std::string CurrName = N;
if (UsedNames.contains(N))
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6708c0f050b72c..b99827321eb03b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2647,16 +2647,13 @@ class VPBasicBlock : public VPBlockBase {
return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
}
- void insert(VPRecipeBase *Recipe, iterator InsertPt) {
- assert(Recipe && "No recipe to append.");
- assert(!Recipe->Parent && "Recipe already in VPlan");
- Recipe->Parent = this;
- Recipes.insert(InsertPt, Recipe);
- }
+ void insert(VPRecipeBase *Recipe, iterator InsertPt);
/// Augment the existing recipes of a VPBasicBlock with an additional
/// \p Recipe as the last recipe.
- void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = "");
+ void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = "") {
+ insert(Recipe, end());
+ }
/// The method which generates the output IR instructions that correspond to
/// this VPBasicBlock, thereby "executing" the VPlan.
@@ -3066,6 +3063,14 @@ class VPlan {
#endif
}
+ bool hasName(const VPValue *V) const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ return VPValue2Name.contains(V);
+#else
+ return false;
+#endif
+ }
+
/// Returns a range mapping the values the range \p Operands to their
/// corresponding VPValues.
iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5b65b1959f9adf..3e63010a37b858 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -198,24 +198,21 @@ void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) {
assert(!Parent && "Recipe already in some VPBasicBlock");
assert(InsertPos->getParent() &&
"Insertion position not in any VPBasicBlock");
- Parent = InsertPos->getParent();
- Parent->getRecipeList().insert(InsertPos->getIterator(), this);
+ InsertPos->getParent()->insert(this, InsertPos->getIterator());
}
void VPRecipeBase::insertBefore(VPBasicBlock &BB,
iplist<VPRecipeBase>::iterator I) {
assert(!Parent && "Recipe already in some VPBasicBlock");
assert(I == BB.end() || I->getParent() == &BB);
- Parent = &BB;
- BB.getRecipeList().insert(I, this);
+ BB.insert(this, I);
}
void VPRecipeBase::insertAfter(VPRecipeBase *InsertPos) {
assert(!Parent && "Recipe already in some VPBasicBlock");
assert(InsertPos->getParent() &&
"Insertion position not in any VPBasicBlock");
- Parent = InsertPos->getParent();
- Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this);
+ InsertPos->getParent()->insert(this, std::next(InsertPos->getIterator()));
}
void VPRecipeBase::removeFromParent() {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6ed3e95d061429..58f3153640e62c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -163,8 +163,6 @@ static bool sinkScalarOperands(VPlan &Plan) {
// TODO: add ".cloned" suffix to name of Clone's VPValue.
Clone->insertBefore(SinkCandidate);
- if (!I->getName().empty())
- Clone->setName(I->getName() + ".cloned");
SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) {
return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
});
>From 180015691116ef22e8a69e6d12b02245313bc7dc Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 14 Mar 2024 11:27:01 +0000
Subject: [PATCH 3/8] fixup
---
.../Transforms/Vectorize/LoopVectorize.cpp | 5 ++--
llvm/lib/Transforms/Vectorize/VPlan.cpp | 14 +++++------
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +---
.../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 7 +++---
.../vplan-printing-before-execute.ll | 24 +++++++++----------
6 files changed, 26 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a16615c04730c9..6141d5454f36b3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8754,11 +8754,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
if (!Member->getType()->isVoidTy()) {
VPValue *OriginalV = MemberR->getVPSingleValue();
OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
+ VPIG->getVPValue(J)->takeName(MemberR->getVPSingleValue());
J++;
}
MemberR->eraseFromParent();
- if (!Member->getType()->isVoidTy())
- VPIG->getVPValue(J - 1)->setName(Member->getName());
}
}
@@ -9016,7 +9015,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
// ensure that it comes after all of it's inputs, including CondOp.
// Note that this transformation may leave over dead recipes (including
// CurrentLink), which will be cleaned by a later VPlan transform.
- LinkVPBB->appendRecipe(RedRecipe, CurrentLinkI->getName());
+ LinkVPBB->appendRecipe(RedRecipe);
CurrentLink->replaceAllUsesWith(RedRecipe);
PreviousLink = RedRecipe;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 459b3b9e3666fc..ba83f39e8c3405 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -612,13 +612,13 @@ void VPBasicBlock::insert(VPRecipeBase *Recipe, iterator InsertPt) {
Recipe->Parent = this;
Recipes.insert(InsertPt, Recipe);
- if (Recipe->getNumDefinedValues() == 1) {
- VPValue *VPV = Recipe->getVPSingleValue();
- Value *UV = VPV->getUnderlyingValue();
- VPlan *ParentPlan = getPlan();
- if (UV && ParentPlan && !ParentPlan->hasName(VPV))
- VPV->setName(UV->getName());
- }
+ if (Recipe->getNumDefinedValues() != 1)
+ return;
+ VPValue *VPV = Recipe->getVPSingleValue();
+ Value *UV = VPV->getUnderlyingValue();
+ VPlan *ParentPlan = getPlan();
+ if (UV && ParentPlan && !ParentPlan->hasName(VPV))
+ VPV->setName(UV->getName());
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b99827321eb03b..0ca25ee0cf1c5e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2651,9 +2651,7 @@ class VPBasicBlock : public VPBlockBase {
/// Augment the existing recipes of a VPBasicBlock with an additional
/// \p Recipe as the last recipe.
- void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = "") {
- insert(Recipe, end());
- }
+ void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
/// The method which generates the output IR instructions that correspond to
/// this VPBasicBlock, thereby "executing" the VPlan.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index eb1c401ed5c8a9..963acaa08ba64c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -309,7 +309,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// an empty VPInstruction that we will fix once the whole plain CFG has
// been built.
NewVPV = new VPWidenPHIRecipe(Phi);
- VPBB->appendRecipe(cast<VPWidenPHIRecipe>(NewVPV), Inst->getName());
+ VPBB->appendRecipe(cast<VPWidenPHIRecipe>(NewVPV));
PhisToFix.push_back(Phi);
} else {
// Translate LLVM-IR operands into VPValue operands and set them in the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3c9fc13a801b0c..84c0f4063d75ae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -160,9 +160,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
Instruction *I = cast<Instruction>(
cast<VPReplicateRecipe>(SinkCandidate)->getUnderlyingValue());
auto *Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true);
- // TODO: add ".cloned" suffix to name of Clone's VPValue.
-
Clone->insertBefore(SinkCandidate);
+ Clone->setName(SinkCandidate->getName() + ".cloned");
SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) {
return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
});
@@ -317,9 +316,9 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
PredRecipe->replaceAllUsesWith(PHIRecipe);
PHIRecipe->setOperand(0, RecipeWithoutMask);
}
+ Plan.takeName(PredRecipe, RecipeWithoutMask);
+
PredRecipe->eraseFromParent();
- Plan.setName(RecipeWithoutMask,
- RecipeWithoutMask->getUnderlyingValue()->getName());
auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true);
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index 1dddbfe20a2ed7..dde5ffc1a6fd3f 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -23,12 +23,12 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: EMIT ir<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1
-; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer ir<%p.src>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]>
-; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
-; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer ir<%p.src>
-; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add>
+; CHECK-NEXT: EMIT vp<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1
+; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<%p.src>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]>
+; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10>
+; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<%p.src>
+; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -54,12 +54,12 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: EMIT ir<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1
-; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer ir<%p.src>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]>
-; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
-; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer ir<%p.src>
-; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add>
+; CHECK-NEXT: EMIT vp<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1
+; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<%p.src>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]>
+; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10>
+; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<%p.src>
+; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: No successors
>From 84117c241b7b9998c2487f786d29d68c9b1d932a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 27 Mar 2024 14:54:52 +0000
Subject: [PATCH 4/8] !fixup merge
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
.../LoopVectorize/AArch64/sve-widen-gep.ll | 16 ++++++-------
.../LoopVectorize/X86/small-size.ll | 24 +++++++++----------
.../vplan-printing-before-execute.ll | 12 +++++-----
.../vplan-sink-scalars-and-merge.ll | 14 +++++------
6 files changed, 38 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d1e9748fa397b1..99f034cfd17d3d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -296,15 +296,17 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
Value *VPInstruction::generatePerLane(VPTransformState &State,
const VPIteration &Lane) {
IRBuilderBase &Builder = State.Builder;
-
assert(getOpcode() == VPInstruction::PtrAdd &&
"only PtrAdd opcodes are supported for now");
+
+std::string Name = getName();
return Builder.CreatePtrAdd(State.get(getOperand(0), Lane),
State.get(getOperand(1), Lane), Name);
}
Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
IRBuilderBase &Builder = State.Builder;
+std::string Name = getName();
if (Instruction::isBinaryOp(getOpcode())) {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b250fd23831437..b6bda9f791febe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -582,10 +582,11 @@ static void legalizeAndOptimizeInductions(VPlan &Plan, ScalarEvolution &SE) {
auto *Recipe =
new VPInstruction(VPInstruction::PtrAdd,
{PtrIV->getStartValue(), Steps->getVPSingleValue()},
- PtrIV->getDebugLoc(), "next.gep");
+ PtrIV->getDebugLoc());
Recipe->insertAfter(Steps);
PtrIV->replaceAllUsesWith(Recipe);
+ Recipe->setName("next.gep");
continue;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index 3bab341e1c248a..ef0027ce4ed9a0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -19,18 +19,18 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
+; CHECK-NEXT: EMIT vp<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
; CHECK-NEXT: vp<[[PTR_IDX:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<8>
; CHECK-NEXT: vp<[[PTR_IDX_STEPS:%.+]]> = SCALAR-STEPS vp<[[PTR_IDX]]>, ir<8>
; CHECK-NEXT: EMIT vp<[[PTR_IV_1:%.+]]> = ptradd ir<%start.1>, vp<[[PTR_IDX_STEPS]]>
-; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1>
+; CHECK-NEXT: WIDEN-GEP Var[Inv] vp<%ptr.iv.2.next> = getelementptr inbounds vp<%ptr.iv.2>, ir<1>
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<[[PTR_IV_1]]>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR2]]>
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1>
-; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer ir<%ptr.iv.2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, ir<%add>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%ptr.iv.2.next>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%ptr.iv.2>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR2]]>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<%lv>, ir<1>
+; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer vp<%ptr.iv.2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
index dc474fbf67ce8b..5fde9a49477129 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -291,7 +291,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]]
-; CHECK: pred.store.if15:
+; CHECK: pred.store.if14:
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 4
@@ -299,10 +299,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16
; CHECK-NEXT: store i32 [[TMP9]], ptr [[NEXT_GEP5]], align 16
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE16]]
-; CHECK: pred.store.continue16:
+; CHECK: pred.store.continue15:
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
-; CHECK: pred.store.if17:
+; CHECK: pred.store.if16:
; CHECK-NEXT: [[TMP11:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 8
@@ -310,10 +310,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP11]], align 16
; CHECK-NEXT: store i32 [[TMP13]], ptr [[NEXT_GEP6]], align 16
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE18]]
-; CHECK: pred.store.continue18:
+; CHECK: pred.store.continue17:
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20]]
-; CHECK: pred.store.if19:
+; CHECK: pred.store.if18:
; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]]
; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 12
@@ -321,7 +321,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP12]], align 16
; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP7]], align 16
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE20]]
-; CHECK: pred.store.continue20:
+; CHECK: pred.store.continue19:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -471,7 +471,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
-; CHECK: pred.store.if12:
+; CHECK: pred.store.if11:
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 4
; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2
@@ -481,10 +481,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7
; CHECK-NEXT: store i32 [[TMP11]], ptr [[NEXT_GEP9]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE13]]
-; CHECK: pred.store.continue13:
+; CHECK: pred.store.continue12:
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
-; CHECK: pred.store.if14:
+; CHECK: pred.store.if13:
; CHECK-NEXT: [[TMP13:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 8
; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
@@ -494,10 +494,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7
; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP10]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE15]]
-; CHECK: pred.store.continue15:
+; CHECK: pred.store.continue14:
; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17]]
-; CHECK: pred.store.if16:
+; CHECK: pred.store.if15:
; CHECK-NEXT: [[TMP19:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 12
; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]]
; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[OFFSET_IDX]], 6
@@ -507,7 +507,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7
; CHECK-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP11]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE17]]
-; CHECK: pred.store.continue17:
+; CHECK: pred.store.continue16:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index ca9dfdc6f6d29d..036386e7dde762 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -26,10 +26,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]>
-; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]>
+; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10>
; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add>
+; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -58,10 +58,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]>
-; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]>
+; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10>
; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add>
+; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 961d83a670c25f..723e7d7baf66ad 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -1116,11 +1116,11 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<%3> * ir<-1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
; CHECK-NEXT: EMIT vp<[[PTR_IV:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>
-; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0>
-; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1>
+; CHECK-NEXT: CLONE vp<%ptr.iv.next.cloned> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%ptr.iv.next.cloned>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%c.1> = icmp eq vp<%l>, ir<0>
+; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not vp<%c.1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -1129,8 +1129,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
-; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next>
+; CHECK-NEXT: REPLICATE vp<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
+; CHECK-NEXT: REPLICATE store ir<95>, vp<%ptr.iv.next>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
>From 529d103194bfe0dcb105076d69d3909d79c718e3 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 3 Apr 2024 15:53:12 +0100
Subject: [PATCH 5/8] !fixup fix formatting
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 40cb72ac219c39..eb85a4fb35106e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -299,14 +299,14 @@ Value *VPInstruction::generatePerLane(VPTransformState &State,
assert(getOpcode() == VPInstruction::PtrAdd &&
"only PtrAdd opcodes are supported for now");
-std::string Name = getName();
+ std::string Name = getName();
return Builder.CreatePtrAdd(State.get(getOperand(0), Lane),
State.get(getOperand(1), Lane), Name);
}
Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
IRBuilderBase &Builder = State.Builder;
-std::string Name = getName();
+ std::string Name = getName();
if (Instruction::isBinaryOp(getOpcode())) {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
>From 857d9258d478df8c03d8ea0efaf5ca125225a66a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 3 Apr 2024 16:12:22 +0100
Subject: [PATCH 6/8] !fixup fix formatting
---
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h | 7 ++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 1c8a2789712a08..8ae9a836663269 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -179,9 +179,10 @@ class VPBuilder {
VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
const Twine &Name = "") {
- return tryInsertInstruction(new VPInstruction(
- Instruction::BinaryOps::Or, {LHS, RHS},
- VPRecipeWithIRFlags::DisjointFlagsTy(false), DL), Name);
+ return tryInsertInstruction(
+ new VPInstruction(Instruction::BinaryOps::Or, {LHS, RHS},
+ VPRecipeWithIRFlags::DisjointFlagsTy(false), DL),
+ Name);
}
VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 845792b9ad30ac..4ab252cfa42da3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1223,8 +1223,7 @@ class VPInstruction : public VPRecipeWithIRFlags {
Opcode(Opcode) {}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- DisjointFlagsTy DisjointFlag, DebugLoc DL = {}
- )
+ DisjointFlagsTy DisjointFlag, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DisjointFlag, DL),
Opcode(Opcode) {
assert(Opcode == Instruction::Or && "only OR opcodes can be disjoint");
>From 8b303322e59b3d13ecf229db1515fdfcfea04777 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 9 Apr 2024 16:55:50 +0100
Subject: [PATCH 7/8] !fixup move deduplication to VPSlotTracker.
---
.../Vectorize/LoopVectorizationPlanner.h | 30 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 133 ++++------
llvm/lib/Transforms/Vectorize/VPlan.h | 84 ++-----
.../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 4 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 15 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 13 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 46 ++--
.../AArch64/sve-tail-folding-forced.ll | 14 +-
.../LoopVectorize/AArch64/sve-widen-gep.ll | 16 +-
.../AArch64/synthesize-mask-for-call.ll | 84 +++----
.../widen-call-with-intrinsic-or-libfunc.ll | 28 +--
.../PowerPC/vplan-force-tail-with-evl.ll | 36 +--
.../RISCV/riscv-vector-reverse.ll | 36 +--
.../LoopVectorize/X86/small-size.ll | 24 +-
.../first-order-recurrence-chains-vplan.ll | 46 ++--
...-order-recurrence-sink-replicate-region.ll | 132 +++++-----
.../Transforms/LoopVectorize/icmp-uniforms.ll | 10 +-
.../interleave-and-scalarize-only.ll | 20 +-
.../LoopVectorize/vplan-dot-printing.ll | 14 +-
.../LoopVectorize/vplan-iv-transforms.ll | 6 +-
.../vplan-printing-before-execute.ll | 12 +-
.../vplan-printing-outer-loop.ll | 26 +-
.../LoopVectorize/vplan-printing.ll | 224 ++++++++---------
.../vplan-sink-scalars-and-merge-vf1.ll | 8 +-
.../vplan-sink-scalars-and-merge.ll | 238 +++++++++---------
.../vplan-unused-interleave-group.ll | 6 +-
27 files changed, 610 insertions(+), 698 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 2c33491898e42a..ece2a34f180cb4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -47,19 +47,16 @@ class VPBuilder {
VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
/// Insert \p VPI in BB at InsertPt if BB is set.
- VPInstruction *tryInsertInstruction(VPInstruction *VPI,
- const Twine &Name = "") {
- if (BB) {
+ VPInstruction *tryInsertInstruction(VPInstruction *VPI) {
+ if (BB)
BB->insert(VPI, InsertPt);
- VPI->setName(Name);
- }
return VPI;
}
VPInstruction *createInstruction(unsigned Opcode,
ArrayRef<VPValue *> Operands, DebugLoc DL,
const Twine &Name = "") {
- return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL), Name);
+ return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
}
VPInstruction *createInstruction(unsigned Opcode,
@@ -162,7 +159,7 @@ class VPBuilder {
VPRecipeWithIRFlags::WrapFlagsTy WrapFlags,
DebugLoc DL = {}, const Twine &Name = "") {
return tryInsertInstruction(
- new VPInstruction(Opcode, Operands, WrapFlags, DL), Name);
+ new VPInstruction(Opcode, Operands, WrapFlags, DL, Name));
}
VPValue *createNot(VPValue *Operand, DebugLoc DL = {},
const Twine &Name = "") {
@@ -177,21 +174,20 @@ class VPBuilder {
VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
const Twine &Name = "") {
- return tryInsertInstruction(
- new VPInstruction(Instruction::BinaryOps::Or, {LHS, RHS},
- VPRecipeWithIRFlags::DisjointFlagsTy(false), DL),
- Name);
+ return tryInsertInstruction(new VPInstruction(
+ Instruction::BinaryOps::Or, {LHS, RHS},
+ VPRecipeWithIRFlags::DisjointFlagsTy(false), DL, Name));
}
VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
DebugLoc DL = {}, const Twine &Name = "",
std::optional<FastMathFlags> FMFs = std::nullopt) {
- auto *Select = FMFs
- ? new VPInstruction(Instruction::Select,
- {Cond, TrueVal, FalseVal}, *FMFs, DL)
- : new VPInstruction(Instruction::Select,
- {Cond, TrueVal, FalseVal}, DL);
- return tryInsertInstruction(Select, Name);
+ auto *Select =
+ FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
+ *FMFs, DL, Name)
+ : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
+ DL, Name);
+ return tryInsertInstruction(Select);
}
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 14a7a558118ee0..5535cc55e93216 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7226,7 +7226,7 @@ VPValue *VPBuilder::createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
return tryInsertInstruction(
- new VPInstruction(Instruction::ICmp, Pred, A, B, DL), Name);
+ new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name));
}
// This function will select a scalable VF if the target supports scalable
@@ -8797,7 +8797,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
if (!Member->getType()->isVoidTy()) {
VPValue *OriginalV = MemberR->getVPSingleValue();
OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
- VPIG->getVPValue(J)->takeName(MemberR->getVPSingleValue());
J++;
}
MemberR->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 7a978909e711aa..c4e7b4d12d3256 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -597,21 +597,6 @@ bool VPBasicBlock::isExiting() const {
return getParent() && getParent()->getExitingBasicBlock() == this;
}
-void VPBasicBlock::insert(VPRecipeBase *Recipe, iterator InsertPt) {
- assert(Recipe && "No recipe to append.");
- assert(!Recipe->Parent && "Recipe already in VPlan");
- Recipe->Parent = this;
- Recipes.insert(InsertPt, Recipe);
-
- if (Recipe->getNumDefinedValues() != 1)
- return;
- VPValue *VPV = Recipe->getVPSingleValue();
- Value *UV = VPV->getUnderlyingValue();
- VPlan *ParentPlan = getPlan();
- if (UV && ParentPlan && !ParentPlan->hasName(VPV))
- VPV->setName(UV->getName());
-}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const {
if (getSuccessors().empty()) {
@@ -1070,10 +1055,8 @@ static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry,
assert(OldR.getNumDefinedValues() == NewR.getNumDefinedValues() &&
"recipes must define the same number of operands");
for (const auto &[OldV, NewV] :
- zip(OldR.definedValues(), NewR.definedValues())) {
+ zip(OldR.definedValues(), NewR.definedValues()))
Old2NewVPValues[OldV] = NewV;
- NewV->setName(OldV->getName());
- }
}
}
@@ -1131,25 +1114,6 @@ VPlan *VPlan::duplicate() {
return NewPlan;
}
-void VPlan::setName(const VPValue *V, const Twine &Name) {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- std::string N = Name.str();
- if (N.empty())
- return;
- std::string CurrName = N;
-
- if (UsedNames.contains(N))
- CurrName = N + ".1";
- unsigned Cnt = 2;
- while (UsedNames.contains(CurrName)) {
- CurrName = N + "." + std::to_string(Cnt);
- Cnt += 1;
- }
- VPValue2Name[V] = CurrName;
- UsedNames.insert(CurrName);
-#endif
-}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1336,12 +1300,7 @@ void VPValue::replaceUsesWithIf(
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const {
- if (!getDefiningRecipe() && getUnderlyingValue()) {
- OS << "ir<";
- getUnderlyingValue()->printAsOperand(OS, false);
- OS << ">";
- } else
- OS << Tracker.getName(this);
+ OS << Tracker.getName(this);
}
void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
@@ -1351,31 +1310,6 @@ void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
}
#endif
-std::string VPValue::getName() {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- if (auto *Def = getDefiningRecipe())
- return Def->getParent()->getPlan()->getName(this);
- assert(isLiveIn() && "called getName() on unsupported VPValue");
- return getLiveInIRValue()->getName().str();
-#else
- return "";
-#endif
-}
-
-void VPValue::setName(const Twine &Name) {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- auto *Def = getDefiningRecipe();
- Def->getParent()->getPlan()->setName(this, Name);
-#endif
-}
-
-void VPValue::takeName(const VPValue *Src) {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- auto *Def = getDefiningRecipe();
- Def->getParent()->getPlan()->takeName(Src, this);
-#endif
-}
-
void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
Old2NewTy &Old2New,
InterleavedAccessInfo &IAI) {
@@ -1428,40 +1362,73 @@ VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
visitRegion(Plan.getVectorLoopRegion(), Old2New, IAI);
}
-void VPSlotTracker::assignSlot(const VPValue *V) {
+void VPSlotTracker::assignSlotOrName(const VPValue *V) {
+ if (auto *UV = V->getUnderlyingValue()) {
+ std::string Name;
+ raw_string_ostream S(Name);
+ UV->printAsOperand(S, false);
+ deduplicateName(V, Name);
+ return;
+ }
assert(!Slots.contains(V) && "VPValue already has a slot!");
Slots[V] = NextSlot++;
}
-void VPSlotTracker::assignSlots(const VPlan &Plan) {
+void VPSlotTracker::assignSlotsOrNames(const VPlan &Plan) {
if (Plan.VFxUF.getNumUsers() > 0)
- assignSlot(&Plan.VFxUF);
- assignSlot(&Plan.VectorTripCount);
+ assignSlotOrName(&Plan.VFxUF);
+ assignSlotOrName(&Plan.VectorTripCount);
if (Plan.BackedgeTakenCount)
- assignSlot(Plan.BackedgeTakenCount);
- assignSlots(Plan.getPreheader());
+ assignSlotOrName(Plan.BackedgeTakenCount);
+ for (VPValue *LI : Plan.VPLiveInsToFree)
+ assignSlotOrName(LI);
+ assignSlotsOrNames(Plan.getPreheader());
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<const VPBlockBase *>>
RPOT(VPBlockDeepTraversalWrapper<const VPBlockBase *>(Plan.getEntry()));
for (const VPBasicBlock *VPBB :
VPBlockUtils::blocksOnly<const VPBasicBlock>(RPOT))
- assignSlots(VPBB);
+ assignSlotsOrNames(VPBB);
}
-void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
+void VPSlotTracker::assignSlotsOrNames(const VPBasicBlock *VPBB) {
for (const VPRecipeBase &Recipe : *VPBB)
for (VPValue *Def : Recipe.definedValues())
- assignSlot(Def);
+ assignSlotOrName(Def);
+}
+
+void VPSlotTracker::deduplicateName(const VPValue *V, StringRef Name) {
+ assert(!Name.empty() && "Name cannot be be empty.");
+ std::string NewName = Name.str();
+ const auto &[A, AssignedInserted] = AssignedNames.insert({V, NewName});
+ if (!AssignedInserted || V->isLiveIn())
+ return;
+
+ const auto &[C, UseInserted] = NameUseCount.insert({NewName, 0});
+ if (!UseInserted) {
+ C->second++;
+ NewName = NewName + "." + std::to_string(C->second);
+ A->second = NewName;
+ }
}
std::string VPSlotTracker::getName(const VPValue *V) const {
- if (!Plan)
- return "<badref>";
- std::string N = Plan->getName(V);
- if (!N.empty())
- return (Twine("vp<%") + N + ">").str();
+ std::string Name = AssignedNames.lookup(V);
+ if (!Name.empty()) {
+ assert(
+ V->getUnderlyingValue() &&
+ "Can only have assigned names for VPValues with an underlying value");
+ return (Twine("ir<") + Name + ">").str();
+ }
- return (Twine("vp<%") + std::to_string(getSlot(V)) + ">").str();
+ assert(
+ !V->getUnderlyingValue() &&
+ "Must not have assigned names for VPValues without an underlying value");
+ unsigned Slot = getSlot(V);
+ if (Slot == unsigned(-1))
+ return "<badref>";
+ else
+ return (Twine("vp<%") + std::to_string(Slot) + ">").str();
}
bool vputils::onlyFirstLaneUsed(const VPValue *Def) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bdff80c2b1285e..5dc905a3c407ba 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -32,7 +32,6 @@
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/ilist.h"
#include "llvm/ADT/ilist_node.h"
@@ -1186,6 +1185,9 @@ class VPInstruction : public VPRecipeWithIRFlags {
typedef unsigned char OpcodeTy;
OpcodeTy Opcode;
+ /// An optional name that can be used for the generated IR instruction.
+ const std::string Name;
+
/// Returns true if this VPInstruction generates scalar values for all lanes.
/// Most VPInstructions generate a single value per part, either vector or
/// scalar. VPReplicateRecipe takes care of generating multiple (scalar)
@@ -1216,37 +1218,39 @@ class VPInstruction : public VPRecipeWithIRFlags {
#endif
public:
- VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL)
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
+ const Twine &Name = "")
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
- Opcode(Opcode) {}
+ Opcode(Opcode), Name(Name.str()) {}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- DebugLoc DL = {})
- : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL) {}
+ DebugLoc DL = {}, const Twine &Name = "")
+ : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}
VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
- VPValue *B, DebugLoc DL = {});
+ VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- WrapFlagsTy WrapFlags, DebugLoc DL = {})
+ WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "")
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL),
- Opcode(Opcode) {}
+ Opcode(Opcode), Name(Name.str()) {}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- DisjointFlagsTy DisjointFlag, DebugLoc DL = {})
+ DisjointFlagsTy DisjointFlag, DebugLoc DL = {},
+ const Twine &Name = "")
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DisjointFlag, DL),
- Opcode(Opcode) {
+ Opcode(Opcode), Name(Name.str()) {
assert(Opcode == Instruction::Or && "only OR opcodes can be disjoint");
}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL = {});
+ FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
VPRecipeBase *clone() override {
SmallVector<VPValue *, 2> Operands(operands());
- auto *New = new VPInstruction(Opcode, Operands, getDebugLoc());
+ auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name);
New->transferFlags(*this);
return New;
}
@@ -2741,7 +2745,12 @@ class VPBasicBlock : public VPBlockBase {
return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
}
- void insert(VPRecipeBase *Recipe, iterator InsertPt);
+ void insert(VPRecipeBase *Recipe, iterator InsertPt) {
+ assert(Recipe && "No recipe to append.");
+ assert(!Recipe->Parent && "Recipe already in VPlan");
+ Recipe->Parent = this;
+ Recipes.insert(InsertPt, Recipe);
+ }
/// Augment the existing recipes of a VPBasicBlock with an additional
/// \p Recipe as the last recipe.
@@ -2964,11 +2973,6 @@ class VPlan {
/// been modeled in VPlan directly.
DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- DenseMap<const VPValue *, std::string> VPValue2Name;
- StringSet<> UsedNames;
-#endif
-
public:
/// Construct a VPlan with original preheader \p Preheader, trip count \p TC
/// and \p Entry to the plan. At the moment, \p Preheader and \p Entry need to
@@ -3099,50 +3103,6 @@ class VPlan {
LLVM_DUMP_METHOD void dump() const;
#endif
- /// Set the name for \p V to \p Name if it is not empty. \p V must not have a
- /// name assigned already.
- void setName(const VPValue *V, const Twine &Name);
-
- /// Remove the assigned name for \p V, if there is one.
- void removeName(const VPValue *V) {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- if (VPValue2Name.contains(V)) {
- UsedNames.erase(VPValue2Name[V]);
- VPValue2Name.erase(V);
- }
-#endif
- }
-
- /// Take the name for \p Src and move it to \p Dst. The name of \p Src will be
- /// empty afterwards.
- void takeName(const VPValue *Src, const VPValue *Dst) {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- std::string N = VPValue2Name.lookup(Src);
- if (N.empty())
- return;
- VPValue2Name[Dst] = N;
- VPValue2Name.erase(Src);
-#endif
- }
-
- /// Return the name assigned to \p V or an empty string if no name has been
- /// assigned.
- std::string getName(const VPValue *V) const {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- return VPValue2Name.lookup(V);
-#else
- return "";
-#endif
- }
-
- bool hasName(const VPValue *V) const {
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- return VPValue2Name.contains(V);
-#else
- return false;
-#endif
- }
-
/// Returns the VPRegionBlock of the vector loop.
VPRegionBlock *getVectorLoopRegion() {
return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index a53f0c9e697536..b57ff2840a7292 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -320,8 +320,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// Build VPInstruction for any arbitrary Instruction without specific
// representation in VPlan.
- NewVPV = cast<VPInstruction>(VPIRBuilder.createNaryOp(
- Inst->getOpcode(), VPOperands, Inst, Inst->getName()));
+ NewVPV = cast<VPInstruction>(
+ VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
}
IRDef2VPValue[Inst] = NewVPV;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e062e27f2f8024..625319954e9b7b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -224,8 +224,6 @@ void VPRecipeBase::removeFromParent() {
iplist<VPRecipeBase>::iterator VPRecipeBase::eraseFromParent() {
assert(getParent() && "Recipe not in any VPBasicBlock");
- for (auto *V : definedValues())
- getParent()->getPlan()->removeName(V);
return getParent()->getRecipeList().erase(getIterator());
}
@@ -255,19 +253,20 @@ FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const {
}
VPInstruction::VPInstruction(unsigned Opcode, CmpInst::Predicate Pred,
- VPValue *A, VPValue *B, DebugLoc DL)
+ VPValue *A, VPValue *B, DebugLoc DL,
+ const Twine &Name)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef<VPValue *>({A, B}),
Pred, DL),
- Opcode(Opcode) {
+ Opcode(Opcode), Name(Name.str()) {
assert(Opcode == Instruction::ICmp &&
"only ICmp predicates supported at the moment");
}
VPInstruction::VPInstruction(unsigned Opcode,
std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL)
+ FastMathFlags FMFs, DebugLoc DL, const Twine &Name)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL),
- Opcode(Opcode) {
+ Opcode(Opcode), Name(Name.str()) {
// Make sure the VPInstruction is a floating-point operation.
assert(isFPMathOp() && "this op can't take fast-math flags");
}
@@ -297,17 +296,15 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
Value *VPInstruction::generatePerLane(VPTransformState &State,
const VPIteration &Lane) {
IRBuilderBase &Builder = State.Builder;
+
assert(getOpcode() == VPInstruction::PtrAdd &&
"only PtrAdd opcodes are supported for now");
-
- std::string Name = getName();
return Builder.CreatePtrAdd(State.get(getOperand(0), Lane),
State.get(getOperand(1), Lane), Name);
}
Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
IRBuilderBase &Builder = State.Builder;
- std::string Name = getName();
if (Instruction::isBinaryOp(getOpcode())) {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e539c491ed9f78..1256e4d8fda50b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -88,10 +88,9 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
}
NewRecipe->insertBefore(&Ingredient);
- if (NewRecipe->getNumDefinedValues() == 1) {
+ if (NewRecipe->getNumDefinedValues() == 1)
VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
- NewRecipe->getVPSingleValue()->takeName(VPV);
- } else
+ else
assert(NewRecipe->getNumDefinedValues() == 0 &&
"Only recpies with zero or one defined values expected");
Ingredient.eraseFromParent();
@@ -160,8 +159,9 @@ static bool sinkScalarOperands(VPlan &Plan) {
continue;
Instruction *I = SinkCandidate->getUnderlyingInstr();
auto *Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true);
+ // TODO: add ".cloned" suffix to name of Clone's VPValue.
+
Clone->insertBefore(SinkCandidate);
- Clone->setName(SinkCandidate->getName() + ".cloned");
SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) {
return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
});
@@ -316,8 +316,6 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
PredRecipe->replaceAllUsesWith(PHIRecipe);
PHIRecipe->setOperand(0, RecipeWithoutMask);
}
- Plan.takeName(PredRecipe, RecipeWithoutMask);
-
PredRecipe->eraseFromParent();
auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true);
@@ -590,11 +588,10 @@ static void legalizeAndOptimizeInductions(VPlan &Plan, ScalarEvolution &SE) {
auto *Recipe =
new VPInstruction(VPInstruction::PtrAdd,
{PtrIV->getStartValue(), Steps->getVPSingleValue()},
- PtrIV->getDebugLoc());
+ PtrIV->getDebugLoc(), "next.gep");
Recipe->insertAfter(Steps);
PtrIV->replaceAllUsesWith(Recipe);
- Recipe->setName("next.gep");
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index a55be92e059a4b..15094c6a60004f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -23,6 +23,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/ADT/iterator_range.h"
@@ -187,21 +188,6 @@ class VPValue {
/// TODO: Also handle recipes defined in pre-header blocks.
bool isDefinedOutsideVectorRegions() const { return !hasDefiningRecipe(); }
- /// Returns the name associated with the VPValue. In case the VPValue has a
- /// defining recipe, look up the name in the containing VPlan. In case it is a
- /// live-in with an underlying IR value, return the name of the underlying
- /// value. Calling getName() in other cases is invalid.
- std::string getName();
-
- /// Sets the name for this VPValue to \p Name. The VPValue must have a
- /// defining recipe inserted in a VPlan.
- void setName(const Twine &Name);
-
- /// Take the name for \p Src and move it to this VPValue. The name of \p Src
- /// will be empty afterwards. The VPValue must have a defining recipe inserted
- /// in a VPlan.
- void takeName(const VPValue *Src);
-
// Set \p Val as the underlying Value of this VPValue.
void setUnderlyingValue(Value *Val) {
assert(!UnderlyingVal && "Underlying Value is already set.");
@@ -458,22 +444,32 @@ class VPDef {
class VPlan;
class VPBasicBlock;
-/// This class can be used to assign consecutive numbers to all VPValues in a
-/// VPlan and allows querying the numbering for printing, similar to the
-/// ModuleSlotTracker for IR values.
+/// This class can be used to assign consecutive numbers to VPValues in a
+/// VPlan without underlying values and deduplicated names to VPValues with
+/// underlying values. Allows querying the numbering and deduplicated names for
+/// printing, similar to the ModuleSlotTracker for IR values.
class VPSlotTracker {
- const VPlan *Plan;
+ /// Keep track of de-duplicated names assigned to VPValues with underlying IR
+ /// values
+ DenseMap<const VPValue *, std::string> AssignedNames;
+ /// Keep track of the next number to use to deduplicate the base name.
+ StringMap<unsigned> NameUseCount;
+
DenseMap<const VPValue *, unsigned> Slots;
unsigned NextSlot = 0;
- void assignSlot(const VPValue *V);
- void assignSlots(const VPlan &Plan);
- void assignSlots(const VPBasicBlock *VPBB);
+ void assignSlotOrName(const VPValue *V);
+ void assignSlotsOrNames(const VPlan &Plan);
+ void assignSlotsOrNames(const VPBasicBlock *VPBB);
+
+ /// Create a deduplicated version of \p Name for \p V by appending ".Number"
+ /// to \p Name if there are multiple uses of that name.
+ void deduplicateName(const VPValue *V, StringRef Name);
public:
- VPSlotTracker(const VPlan *Plan = nullptr) : Plan(Plan) {
+ VPSlotTracker(const VPlan *Plan = nullptr) {
if (Plan)
- assignSlots(*Plan);
+ assignSlotsOrNames(*Plan);
}
unsigned getSlot(const VPValue *V) const {
@@ -483,7 +479,7 @@ class VPSlotTracker {
return I->second;
}
- std::string getName(const VPValue *V) const;
+ std::string getName(const VPValue *) const;
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
index 33708bad471da3..cc72dfa4ce639f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
@@ -19,20 +19,20 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]>
-; VPLANS-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[VF:%[0-9]+]]> = VF * Part + ir<0>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%[0-9]+]]> = active lane mask vp<[[VF]]>, vp<[[TC]]>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
; VPLANS-NEXT: <x1> vector loop: {
; VPLANS-NEXT: vector.body:
; VPLANS-NEXT: EMIT vp<[[INDV:%[0-9]+]]> = CANONICAL-INDUCTION
-; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<[[LANEMASK_LOOP:%.+]]>
+; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%[0-9]+]]>
; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1>
-; VPLANS-NEXT: CLONE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
-; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer vp<%gep>
+; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
+; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer ir<%gep>
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
-; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
-; VPLANS-NEXT: EMIT vp<[[INC:%.+]]> = VF * Part + vp<[[INDV]]>
+; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
+; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[NEWTC]]>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index ef0027ce4ed9a0..3bab341e1c248a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -19,18 +19,18 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: EMIT vp<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
+; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
; CHECK-NEXT: vp<[[PTR_IDX:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<8>
; CHECK-NEXT: vp<[[PTR_IDX_STEPS:%.+]]> = SCALAR-STEPS vp<[[PTR_IDX]]>, ir<8>
; CHECK-NEXT: EMIT vp<[[PTR_IV_1:%.+]]> = ptradd ir<%start.1>, vp<[[PTR_IDX_STEPS]]>
-; CHECK-NEXT: WIDEN-GEP Var[Inv] vp<%ptr.iv.2.next> = getelementptr inbounds vp<%ptr.iv.2>, ir<1>
+; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1>
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<[[PTR_IV_1]]>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%ptr.iv.2.next>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%ptr.iv.2>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR2]]>
-; CHECK-NEXT: WIDEN vp<%add> = add vp<%lv>, ir<1>
-; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer vp<%ptr.iv.2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, vp<%add>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR2]]>
+; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1>
+; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer ir<%ptr.iv.2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, ir<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index 02e348a8e6f57b..ad0d40ee7fbde9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -21,13 +21,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
-; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REPLICATE vp<%call> = call @foo(vp<%load>)
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -50,13 +50,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
-; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask)
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -84,13 +84,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
-; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask)
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%call>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -113,13 +113,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
-; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>, ir<true>) (using library function: foo_vector_fixed4_mask)
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir<true>) (using library function: foo_vector_fixed4_mask)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -146,13 +146,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
-; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask)
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -175,13 +175,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
-; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask)
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
index 9b8cbb37ea2de7..f67b932b04a0d6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
@@ -19,13 +19,13 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src>
-; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double
-; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using library function: __simd_sin_v2f64)
-; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst>
+; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double
+; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64)
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -48,13 +48,13 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src>
-; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double
-; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using vector intrinsic)
-; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst>
+; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double
+; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic)
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
index 5d1a471c5d166f..bd52c2a8f06452 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
@@ -18,37 +18,37 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%16>
+; CHECK-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<%5> = icmp ule ir<%iv>, vp<%2>
+; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv>, vp<%2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<%5>
+; CHECK-NEXT: BRANCH-ON-MASK vp<%4>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: vp<%6> = SCALAR-STEPS vp<%3>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6>
+; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%3>, ir<1>
+; CHECK-NEXT: REPLICATE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%5>
; CHECK-NEXT: REPLICATE ir<%0> = load ir<%arrayidx>
-; CHECK-NEXT: REPLICATE ir<%arrayidx2> = getelementptr inbounds ir<%c>, vp<%6>
+; CHECK-NEXT: REPLICATE ir<%arrayidx2> = getelementptr inbounds ir<%c>, vp<%5>
; CHECK-NEXT: REPLICATE ir<%1> = load ir<%arrayidx2>
-; CHECK-NEXT: REPLICATE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6>
+; CHECK-NEXT: REPLICATE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%5>
; CHECK-NEXT: REPLICATE ir<%add> = add nsw ir<%1>, ir<%0>
; CHECK-NEXT: REPLICATE store ir<%add>, ir<%arrayidx4>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%14> = ir<%0>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%15> = ir<%1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%0>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%7> = ir<%1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): for.body.2
; CHECK-EMPTY:
; CHECK-NEXT: for.body.2:
-; CHECK-NEXT: EMIT vp<%16> = add vp<%3>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%16>, vp<%1>
+; CHECK-NEXT: EMIT vp<%8> = add vp<%3>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%8>, vp<%1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
@@ -83,17 +83,17 @@ define void @safe_dep(ptr %p) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%10>
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6>
; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
; CHECK-NEXT: CLONE ir<%a1> = getelementptr ir<%p>, vp<%3>
-; CHECK-NEXT: vp<%5> = vector-pointer ir<%a1>
-; CHECK-NEXT: WIDEN ir<%v> = load vp<%5>
+; CHECK-NEXT: vp<%4> = vector-pointer ir<%a1>
+; CHECK-NEXT: WIDEN ir<%v> = load vp<%4>
; CHECK-NEXT: CLONE ir<%offset> = add vp<%3>, ir<100>
; CHECK-NEXT: CLONE ir<%a2> = getelementptr ir<%p>, ir<%offset>
-; CHECK-NEXT: vp<%9> = vector-pointer ir<%a2>
-; CHECK-NEXT: WIDEN store vp<%9>, ir<%v>
-; CHECK-NEXT: EMIT vp<%10> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%10>, vp<%1>
+; CHECK-NEXT: vp<%5> = vector-pointer ir<%a2>
+; CHECK-NEXT: WIDEN store vp<%5>, ir<%v>
+; CHECK-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%6>, vp<%1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 5440c5324a42bb..72d9691b2bb877 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -64,15 +64,15 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0>
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx>
-; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%add9> = add vp<[[L]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add9>
+; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -205,15 +205,15 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0>
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx>
-; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%conv1> = fadd vp<[[L]]>, ir<1.000000e+00>
-; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%conv1>
+; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
+; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1>
; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
index 5fde9a49477129..dc474fbf67ce8b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -291,7 +291,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]]
-; CHECK: pred.store.if14:
+; CHECK: pred.store.if15:
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 4
@@ -299,10 +299,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16
; CHECK-NEXT: store i32 [[TMP9]], ptr [[NEXT_GEP5]], align 16
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE16]]
-; CHECK: pred.store.continue15:
+; CHECK: pred.store.continue16:
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
-; CHECK: pred.store.if16:
+; CHECK: pred.store.if17:
; CHECK-NEXT: [[TMP11:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 8
@@ -310,10 +310,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP11]], align 16
; CHECK-NEXT: store i32 [[TMP13]], ptr [[NEXT_GEP6]], align 16
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE18]]
-; CHECK: pred.store.continue17:
+; CHECK: pred.store.continue18:
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20]]
-; CHECK: pred.store.if18:
+; CHECK: pred.store.if19:
; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]]
; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 12
@@ -321,7 +321,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP12]], align 16
; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP7]], align 16
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE20]]
-; CHECK: pred.store.continue19:
+; CHECK: pred.store.continue20:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -471,7 +471,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
-; CHECK: pred.store.if11:
+; CHECK: pred.store.if12:
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 4
; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2
@@ -481,10 +481,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7
; CHECK-NEXT: store i32 [[TMP11]], ptr [[NEXT_GEP9]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE13]]
-; CHECK: pred.store.continue12:
+; CHECK: pred.store.continue13:
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
-; CHECK: pred.store.if13:
+; CHECK: pred.store.if14:
; CHECK-NEXT: [[TMP13:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 8
; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
@@ -494,10 +494,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7
; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP10]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE15]]
-; CHECK: pred.store.continue14:
+; CHECK: pred.store.continue15:
; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17]]
-; CHECK: pred.store.if15:
+; CHECK: pred.store.if16:
; CHECK-NEXT: [[TMP19:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 12
; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]]
; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[OFFSET_IDX]], 6
@@ -507,7 +507,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7
; CHECK-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP11]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE17]]
-; CHECK: pred.store.continue16:
+; CHECK: pred.store.continue17:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index 7db4d65df24291..c04178a1c13e21 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -15,17 +15,17 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr>
-; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next>
-; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]>
-; CHECK-NEXT: WIDEN vp<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
+; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr>
+; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next>
+; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]>
+; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -68,20 +68,20 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr>
-; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next>
-; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]>
-; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice vp<%for.3>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: WIDEN vp<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: WIDEN vp<%add.2> = add vp<%add.1>, vp<[[FOR3_SPLICE]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add.2>
+; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr>
+; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next>
+; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]>
+; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice ir<%for.3>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add.2>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 06f7a7be352f4b..833d55f09294e8 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -20,10 +20,10 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -32,19 +32,19 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V)
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED1]]> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED1]]> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -53,14 +53,14 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%add> = add vp<%conv>, vp<%rem>
-; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst>
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem>
+; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -111,11 +111,11 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -124,15 +124,15 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%add> = add vp<%rem>, vp<%recur.next>
-; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next>
+; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
@@ -181,12 +181,12 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%and.red> = phi ir<1234>, vp<%and.red.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
+; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: Successor(s): pred.srem
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.srem: {
@@ -195,19 +195,19 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.if:
-; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
; CHECK-NEXT: Successor(s): pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, vp<%recur.next>
-; CHECK-NEXT: WIDEN vp<%and.red.next> = and vp<%and.red>, vp<%add>
-; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, vp<%and.red.next>, vp<%and.red>
+; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next>
+; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add>
+; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -215,7 +215,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%and.red>, vp<[[SEL]]>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]>
@@ -257,11 +257,11 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -270,18 +270,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V)
+; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED]]> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED]]> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK: <xVFxUF> pred.store: {
@@ -290,18 +290,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE vp<%lv.2> = load vp<%gep>
-; CHECK-NEXT: REPLICATE vp<%conv.lv.2> = sext vp<%lv.2>
-; CHECK-NEXT: REPLICATE vp<%add.1> = add vp<%conv>, vp<%rem>
-; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%add> = add vp<%add.1>, vp<%conv.lv.2>
-; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst>
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep>
+; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2>
+; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem>
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2>
+; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%rem>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.2>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
@@ -360,11 +360,11 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -374,17 +374,17 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE vp<%rem.div> = sdiv ir<20>, vp<%rem>
-; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep>
-; CHECK-NEXT: REPLICATE vp<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep.2>
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep>
+; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem.div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem.div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
@@ -435,12 +435,12 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%.pn> = phi ir<0>, vp<[[L:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]>, vp<[[BTC]]>
-; CHECK-NEXT: CLONE vp<[[L]]> = load ir<%src>
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%.pn>, vp<[[L]]>
+; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn>, ir<[[L]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -449,14 +449,14 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%val> = sdiv vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x>
; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%val>, vp<%gep.dst>
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = vp<%val>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index f0b8ded142210d..b4ab6f7e8ceb70 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -49,9 +49,9 @@ for.end:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN vp<%cond0> = icmp ult vp<%iv>, ir<13>
-; CHECK-NEXT: WIDEN-SELECT vp<%s> = select vp<%cond0>, ir<10>, ir<20>
+; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13>
+; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -61,8 +61,8 @@ for.end:
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index 54a66dd424faed..81cc2024bb31a5 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -21,9 +21,9 @@
; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%start> + vp<[[CAN_IV]]> * ir<1>
; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; DBG-NEXT: CLONE vp<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>)
-; DBG-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]>
-; DBG-NEXT: CLONE store vp<%min>, vp<%arrayidx>
+; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>)
+; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]>
+; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx>
; DBG-NEXT: EMIT vp<[[INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; DBG-NEXT: EMIT branch-on-count vp<[[INC]]>, vp<[[VEC_TC]]>
; DBG-NEXT: No successors
@@ -89,14 +89,14 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-EMPTY:
; DBG-NEXT: pred.store.if:
; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; DBG-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]>
-; DBG-NEXT: CLONE vp<%l> = load vp<%gep.src>
-; DBG-NEXT: CLONE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]>
-; DBG-NEXT: CLONE store vp<%l>, vp<%gep.dst>
+; DBG-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]>
+; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src>
+; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]>
+; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst>
; DBG-NEXT: Successor(s): pred.store.continue
; DBG-EMPTY:
; DBG-NEXT: pred.store.continue:
-; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = vp<%l>
+; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l>
; DBG-NEXT: No successors
; DBG-NEXT: }
; DBG-NEXT: Successor(s): cond.false.1
@@ -190,10 +190,10 @@ exit:
; DBG-NEXT: <x1> vector loop: {
; DBG-NEXT: vector.body:
; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]>
+; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]>
; DBG-NEXT: SCALAR-CAST vp<[[TRUNC_IV:%.+]]> = trunc vp<[[CAN_IV]]> to i32
; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[TRUNC_IV]]>, vp<[[CAST]]>
-; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[SCALAR_STEPS]]>
+; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[SCALAR_STEPS]]>
; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst>
; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VTC]]>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index 85d035f1ddaee0..12b8e657aabf12 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -28,13 +28,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: "vector.body:\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION ir\<0\>, vp\<[[CAN_IV_NEXT:%.+]]\>\l" +
; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" +
-; CHECK-NEXT: " CLONE vp\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer vp\<%arrayidx\>\l" +
-; CHECK-NEXT: " WIDEN vp\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
-; CHECK-NEXT: " WIDEN-CALL vp\<%call\> = call @llvm.sqrt.f32(vp\<%lv\>) (using vector intrinsic)\l" +
-; CHECK-NEXT: " CLONE vp\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer vp\<%arrayidx2\>\l" +
-; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, vp\<%call\>\l" +
+; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer ir\<%arrayidx\>\l" +
+; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
+; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" +
+; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer ir\<%arrayidx2\>\l" +
+; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" +
; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" +
; CHECK-NEXT: "No successors\l"
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 00996e7eee5664..7ab2459ada2ed6 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -16,9 +16,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next.p, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%iv>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%iv>
; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index 036386e7dde762..ca9dfdc6f6d29d 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -26,10 +26,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]>
-; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]>
+; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add>
+; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -58,10 +58,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]>
-; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]>
+; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10>
; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]>
-; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add>
+; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 1795c6a4fe33f5..65df5973ac75b6 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -15,28 +15,28 @@ define void @foo(i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-PHI vp<%outer.iv> = phi ir<0>, vp<%outer.iv.next>
-; CHECK-NEXT: EMIT vp<%gep.1> = getelementptr ir<@arr2>, ir<0>, vp<%outer.iv>
-; CHECK-NEXT: EMIT store vp<%outer.iv>, vp<%gep.1>
-; CHECK-NEXT: EMIT vp<%add> = add vp<%outer.iv>, ir<%n>
+; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
+; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
+; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1>
+; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
; CHECK-NEXT: Successor(s): inner
; CHECK-EMPTY:
; CHECK-NEXT: <x1> inner: {
; CHECK-NEXT: inner:
-; CHECK-NEXT: WIDEN-PHI vp<%inner.iv> = phi ir<0>, vp<%inner.iv.next>
-; CHECK-NEXT: EMIT vp<%gep.2> = getelementptr ir<@arr>, ir<0>, vp<%inner.iv>, vp<%outer.iv>
-; CHECK-NEXT: EMIT store vp<%add>, vp<%gep.2>
-; CHECK-NEXT: EMIT vp<%inner.iv.next> = add vp<%inner.iv>, ir<1>
-; CHECK-NEXT: EMIT vp<%inner.ec> = icmp vp<%inner.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond vp<%inner.ec>
+; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
+; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
+; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2>
+; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
+; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): outer.latch
; CHECK-EMPTY:
; CHECK-NEXT: outer.latch:
-; CHECK-NEXT: EMIT vp<%outer.iv.next> = add vp<%outer.iv>, ir<1>
-; CHECK-NEXT: EMIT vp<%outer.ec> = icmp vp<%outer.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond vp<%outer.ec>
+; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
+; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 56431af1b854ee..7056bbe6ba1b77 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -20,13 +20,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL vp<%call> = call @llvm.sqrt.f32(vp<%lv>)
-; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -71,15 +71,15 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: WIDEN-GEP Inv[Var] vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<%iv>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%cmp> = icmp eq vp<%arrayidx>, ir<%z>
-; CHECK-NEXT: WIDEN-SELECT vp<%sel> = select vp<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
-; CHECK-NEXT: WIDEN vp<%add> = fadd vp<%lv>, vp<%sel>
-; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
+; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr inbounds ir<%y>, ir<%iv>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z>
+; CHECK-NEXT: WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
+; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -124,12 +124,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -137,7 +137,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%red>, ir<%red.next>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out float %red.next.lcssa = vp<[[RED_RES]]>
@@ -173,12 +173,12 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>) (with final reduction value stored in invariant address sank outside of loop)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -186,7 +186,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1>
+; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result ir<%red>, ir<%red.next>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
@@ -227,20 +227,20 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%i>, ir<5>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5>
; CHECK-NEXT: Successor(s): pred.udiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.udiv: {
; CHECK-NEXT: pred.udiv.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<%cmp>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp>
; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.if:
-; CHECK-NEXT: REPLICATE vp<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
+; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
; CHECK-NEXT: Successor(s): pred.udiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%tmp4>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%tmp4>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.0
@@ -302,19 +302,19 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4>
-; CHECK-NEXT: CLONE vp<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, vp<%gep.AB.0>
-; CHECK-NEXT: vp<%AB.0> = load from index 0
-; CHECK-NEXT: vp<%AB.1> = load from index 1
-; CHECK-NEXT: vp<%AB.3> = load from index 3
-; CHECK-NEXT: CLONE vp<%iv.plus.3> = add vp<[[STEPS]]>, ir<3>
-; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%AB.0>, vp<%AB.1>
-; CHECK-NEXT: CLONE vp<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, vp<%iv.plus.3>
-; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at <badref>, vp<%gep.CD.3>
-; CHECK-NEXT: store vp<%add> to index 0
+; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0>
+; CHECK-NEXT: ir<%AB.0> = load from index 0
+; CHECK-NEXT: ir<%AB.1> = load from index 1
+; CHECK-NEXT: ir<%AB.3> = load from index 3
+; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<[[STEPS]]>, ir<3>
+; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%AB.0>, ir<%AB.1>
+; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, ir<%iv.plus.3>
+; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%gep.CD.3>
+; CHECK-NEXT: store ir<%add> to index 0
; CHECK-NEXT: store ir<1> to index 1
; CHECK-NEXT: store ir<2> to index 2
-; CHECK-NEXT: store vp<%AB.3> to index 3
+; CHECK-NEXT: store ir<%AB.3> to index 3
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -369,16 +369,16 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%sum.07> = phi ir<0.000000e+00>, vp<%muladd.1>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
-; CHECK-NEXT: WIDEN vp<%l.a> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
-; CHECK-NEXT: WIDEN vp<%l.b> = load vp<[[VEC_PTR2]]>
-; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz vp<%l.a>, vp<%l.b>
-; CHECK-NEXT: REDUCE vp<[[MULADD:%.+]]> = vp<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
+; CHECK-NEXT: WIDEN ir<%l.b> = load vp<[[VEC_PTR2]]>
+; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b>
+; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -386,7 +386,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%sum.07>, vp<[[MULADD]]>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%sum.07>, ir<[[MULADD]]>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out float %muladd.lcssa = vp<[[RED_RES]]>
@@ -425,15 +425,15 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%isd>
-; CHECK-NEXT: WIDEN vp<%lsd> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%psd> = add nuw nsw vp<%lsd>, ir<23>
-; CHECK-NEXT: WIDEN vp<%cmp1> = icmp slt vp<%lsd>, ir<100>
-; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not vp<%cmp1>, !dbg /tmp/s.c:5:3
-; CHECK-NEXT: WIDEN vp<%cmp2> = icmp sge vp<%lsd>, ir<200>
-; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, vp<%cmp2>, ir<false>, !dbg /tmp/s.c:5:21
-; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, vp<%cmp1>
+; CHECK-NEXT: CLONE ir<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%isd>
+; CHECK-NEXT: WIDEN ir<%lsd> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%psd> = add nuw nsw ir<%lsd>, ir<23>
+; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100>
+; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3
+; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200>
+; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, ir<%cmp2>, ir<false>, !dbg /tmp/s.c:5:21
+; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, ir<%cmp1>
; CHECK-NEXT: Successor(s): pred.sdiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.sdiv: {
@@ -442,17 +442,17 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.if:
-; CHECK-NEXT: REPLICATE vp<%sd1> = sdiv vp<%psd>, vp<%lsd> (S->V)
+; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V)
; CHECK-NEXT: Successor(s): pred.sdiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = vp<%sd1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = ir<%sd1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
-; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp2>
+; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2>
; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir<false>
; CHECK-NEXT: BLEND ir<%ysd.0> = vp<[[PHI]]> ir<%psd>/vp<[[SEL2]]>
; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%isd>
@@ -520,12 +520,12 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION\l" +
; CHECK-NEXT: " %iv = phi %iv.next, 0\l" +
-; CHECK-NEXT: " vp<%v2>, vp<[[EXP_SCEV]]>
+; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]>
-; CHECK-NEXT: WIDEN vp<%v3> = add nuw vp<%v2>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%v3>, vp<%gep>
+; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -571,9 +571,9 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: WIDEN vp<%add> = add vp<%iv>, ir<%off>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<0>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
@@ -584,7 +584,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: Live-out i32 %lcssa = vp<%add>
+; CHECK-NEXT: Live-out i32 %lcssa = ir<%add>
; CHECK-NEXT: }
;
entry:
@@ -618,15 +618,15 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.y>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%add> = fadd nnan vp<%lv>, ir<1.000000e+00>
-; CHECK-NEXT: WIDEN vp<%mul> = fmul reassoc nnan ninf nsz arcp contract afn vp<%add>, ir<2.000000e+00>
-; CHECK-NEXT: WIDEN vp<%div> = fdiv reassoc nsz contract vp<%mul>, ir<2.000000e+00>
-; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%div>
+; CHECK-NEXT: CLONE ir<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.y>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%add> = fadd nnan ir<%lv>, ir<1.000000e+00>
+; CHECK-NEXT: WIDEN ir<%mul> = fmul reassoc nnan ninf nsz arcp contract afn ir<%add>, ir<2.000000e+00>
+; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00>
+; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%div>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -671,14 +671,14 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%div.1> = udiv exact vp<%lv>, ir<20>
-; CHECK-NEXT: WIDEN vp<%div.2> = udiv vp<%lv>, ir<60>
-; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%div.1>, vp<%div.2>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
+; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%div.1> = udiv exact ir<%lv>, ir<20>
+; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60>
+; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -722,25 +722,25 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ld.addr>
-; CHECK-NEXT: WIDEN vp<%ld.value> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%ifcond> = fcmp oeq vp<%ld.value>, ir<5.000000e+00>
+; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ld.addr>
+; CHECK-NEXT: WIDEN ir<%ld.value> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%ifcond> = fcmp oeq ir<%ld.value>, ir<5.000000e+00>
; CHECK-NEXT: Successor(s): pred.call
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.call: {
; CHECK-NEXT: pred.call.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<%ifcond>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%ifcond>
; CHECK-NEXT: Successor(s): pred.call.if, pred.call.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.call.if:
-; CHECK-NEXT: REPLICATE vp<%foo.ret.1> = call nnan ninf nsz @foo(vp<%ld.value>) (S->V)
-; CHECK-NEXT: REPLICATE vp<%foo.ret.2> = call @foo(vp<%ld.value>) (S->V)
+; CHECK-NEXT: REPLICATE ir<%foo.ret.1> = call nnan ninf nsz @foo(ir<%ld.value>) (S->V)
+; CHECK-NEXT: REPLICATE ir<%foo.ret.2> = call @foo(ir<%ld.value>) (S->V)
; CHECK-NEXT: Successor(s): pred.call.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.call.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = vp<%foo.ret.1>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = vp<%foo.ret.2>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = ir<%foo.ret.1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = ir<%foo.ret.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.1
@@ -804,14 +804,14 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
-; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%or.1> = or disjoint vp<%lv>, ir<1>
-; CHECK-NEXT: WIDEN vp<%or.2> = or vp<%lv>, ir<3>
-; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%or.1>, vp<%or.2>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%add>
+; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%or.1> = or disjoint ir<%lv>, ir<1>
+; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3>
+; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -855,11 +855,11 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%p>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx>
-; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST vp<%zext> = zext nneg vp<%l>
-; CHECK-NEXT: REPLICATE store vp<%zext>, ir<%p1>
+; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l>
+; CHECK-NEXT: REPLICATE store ir<%zext>, ir<%p1>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index 7bf5e6bd76bd3d..8c18c107b39fc1 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -19,9 +19,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: CLONE vp<%tmp3> = load vp<%tmp2>
-; CHECK-NEXT: CLONE store ir<0>, vp<%tmp2>
+; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2>
+; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -30,7 +30,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: CLONE store vp<%tmp3>, vp<%tmp2>
+; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 0e19eb5a23050a..06bd2e21470328 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -27,7 +27,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -37,16 +37,16 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
-; CHECK-NEXT: REPLICATE vp<%add> = add vp<%lv.b>, ir<10>
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]
-; CHECK-NEXT: REPLICATE vp<%mul> = mul ir<2>, vp<%add>
-; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
+; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]
+; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add>
+; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -96,7 +96,7 @@ exit:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK: <xVFxUF> pred.load: {
@@ -106,17 +106,17 @@ exit:
; CHECK: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: loop.0:
-; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<2>
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -125,9 +125,9 @@ exit:
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
-; CHECK-NEXT: REPLICATE vp<%add> = add vp<[[PRED]]>, ir<10>
-; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
+; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10>
+; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
@@ -180,7 +180,7 @@ exit:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK: <xVFxUF> pred.load: {
@@ -190,18 +190,18 @@ exit:
; CHECK: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: loop.0:
-; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, ir<10>
-; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, vp<%add>
+; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<10>
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -210,8 +210,8 @@ exit:
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
-; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
+; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
@@ -265,10 +265,10 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]>, vp<[[BTC]]>
-; CHECK-NEXT: CLONE vp<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0>
-; CHECK-NEXT: CLONE vp<%lv> = load vp<%gep.A.uniform>
-; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%iv>, ir<%k>
-; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp>
+; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0>
+; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k>
+; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp>
; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<[[NOT2]]>, ir<false>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
@@ -279,8 +279,8 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%lv>, vp<%gep.B>
+; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -337,10 +337,10 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN vp<%c.1> = icmp ult vp<%iv>, ir<%j>
-; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.1>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j>
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.1>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -350,18 +350,18 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.1>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.1>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]>
@@ -373,8 +373,8 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
-; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
+; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -437,11 +437,11 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
-; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
-; CHECK-NEXT: WIDEN vp<%c.1> = icmp ugt vp<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.0>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
+; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
+; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -451,18 +451,18 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]>
@@ -475,8 +475,8 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
-; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
+; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -546,10 +546,10 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
-; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, vp<%c.0>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
+; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, ir<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -559,18 +559,18 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]>
@@ -583,10 +583,10 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
-; CHECK-NEXT: REPLICATE store ir<0>, vp<%gep.a>
-; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<%mul>
-; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.c>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
+; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, ir<%mul>
+; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -656,7 +656,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -665,26 +665,26 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
-; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
-; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%lv.a>, vp<%gep.c>
-; CHECK-NEXT: REPLICATE store vp<%lv.b>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
+; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c>
+; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
; CHECK-EMPTY:
; CHECK-NEXT: loop.3:
-; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<%c.0>, ir<false>
-; CHECK-NEXT: WIDEN vp<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>
+; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -693,8 +693,8 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.c.1>
+; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -760,7 +760,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -770,15 +770,15 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
-; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<%lv.a>, vp<%lv.a>
-; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a>
+; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -827,10 +827,10 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[PRED:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -839,17 +839,17 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = vp<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[PRED]]>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[PRED]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -858,12 +858,12 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>
-; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>
+; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -915,15 +915,15 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%l1> = load ir<%src>
-; CHECK-NEXT: REPLICATE vp<%l2> = trunc vp<%l1>
-; CHECK-NEXT: REPLICATE vp<%cmp> = icmp eq vp<%l1>, ir<0>
-; CHECK-NEXT: REPLICATE vp<%sel> = select vp<%cmp>, ir<5>, vp<%l2>
-; CHECK-NEXT: REPLICATE store vp<%sel>, ir<%dst>
+; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
+; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
+; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
+; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
+; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%l1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%l1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.then.1
@@ -973,11 +973,11 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE vp<%gep.cloned> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.cloned>
-; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%pred> = fcmp oeq vp<[[L]]>, ir<0.000000e+00>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not vp<%pred>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%0> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not ir<%pred>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, vp<%gep>
+; CHECK-NEXT: REPLICATE ir<%gep.1> = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep.1>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -1055,14 +1055,14 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: REPLICATE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE vp<%l> = load vp<%gep.src>
-; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE store vp<%l>, vp<%gep.dst>
+; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src>
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = vp<%l>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = ir<%l>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
@@ -1116,11 +1116,11 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<%3> * ir<-1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
; CHECK-NEXT: EMIT vp<[[PTR_IV:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>
-; CHECK-NEXT: CLONE vp<%ptr.iv.next.cloned> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%ptr.iv.next.cloned>
-; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN vp<%c.1> = icmp eq vp<%l>, ir<0>
-; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not vp<%c.1>
+; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0>
+; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -1129,8 +1129,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE vp<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
-; CHECK-NEXT: REPLICATE store ir<95>, vp<%ptr.iv.next>
+; CHECK-NEXT: REPLICATE ir<%ptr.iv.next.1> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
+; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next.1>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
index 27d81de260d3b9..5ea27994b356da 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
@@ -18,9 +18,9 @@ define void @test_unused_interleave(ptr %src, i32 %length) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1>
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%3>
+; CHECK-NEXT: EMIT vp<%3> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%3>, vp<%1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
>From 55da2ab24b500f4e79a54514c0c212f5cd8eb938 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 12 Apr 2024 13:34:10 +0200
Subject: [PATCH 8/8] !fixup address latest comments, thanks!
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 69 +++++++++++--------
llvm/lib/Transforms/Vectorize/VPlanValue.h | 34 +++++----
.../vplan-sink-scalars-and-merge.ll | 8 +--
3 files changed, 60 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index c4e7b4d12d3256..d3c9dbb0889413 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1363,15 +1363,17 @@ VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
}
void VPSlotTracker::assignSlotOrName(const VPValue *V) {
- if (auto *UV = V->getUnderlyingValue()) {
- std::string Name;
- raw_string_ostream S(Name);
- UV->printAsOperand(S, false);
- deduplicateName(V, Name);
+ auto *UV = V->getUnderlyingValue();
+ if (!UV) {
+ assert(!VPValue2Name.contains(V) && "VPValue already has a slot!");
+ VPValue2Name[V] = (Twine("vp<%") + Twine(NextSlot) + ">").str();
+ NextSlot++;
return;
}
- assert(!Slots.contains(V) && "VPValue already has a slot!");
- Slots[V] = NextSlot++;
+ std::string Name;
+ raw_string_ostream S(Name);
+ UV->printAsOperand(S, false);
+ versionName(V, Name);
}
void VPSlotTracker::assignSlotsOrNames(const VPlan &Plan) {
@@ -1397,38 +1399,47 @@ void VPSlotTracker::assignSlotsOrNames(const VPBasicBlock *VPBB) {
assignSlotOrName(Def);
}
-void VPSlotTracker::deduplicateName(const VPValue *V, StringRef Name) {
- assert(!Name.empty() && "Name cannot be be empty.");
- std::string NewName = Name.str();
- const auto &[A, AssignedInserted] = AssignedNames.insert({V, NewName});
- if (!AssignedInserted || V->isLiveIn())
+void VPSlotTracker::versionName(const VPValue *V, StringRef Name) {
+ assert(!Name.empty() && "Name cannot be empty.");
+ std::string BaseName = (Twine("ir<") + Name + Twine(">")).str();
+
+ // First assign the base name for V.
+ const auto &[A, AssignedInserted] = VPValue2Name.insert({V, BaseName});
+ assert(AssignedInserted && "name assigned already?");
+ if (V->isLiveIn())
return;
- const auto &[C, UseInserted] = NameUseCount.insert({NewName, 0});
+ // If it is already used by C > 0 other VPValues, increase the version counter
+ // C and use it for V.
+ const auto &[C, UseInserted] = BaseName2Version.insert({BaseName, 0});
if (!UseInserted) {
C->second++;
- NewName = NewName + "." + std::to_string(C->second);
- A->second = NewName;
+ A->second = (BaseName + Twine(".") + Twine(C->second)).str();
}
}
std::string VPSlotTracker::getName(const VPValue *V) const {
- std::string Name = AssignedNames.lookup(V);
- if (!Name.empty()) {
- assert(
- V->getUnderlyingValue() &&
- "Can only have assigned names for VPValues with an underlying value");
- return (Twine("ir<") + Name + ">").str();
+ std::string Name = VPValue2Name.lookup(V);
+ if (!Name.empty())
+ return Name;
+
+ // If no name was assigned, no VPlan was provided when creating the slot
+ // tracker or it is not reachable from the provided VPlan. This can happen,
+ // e.g. when trying to print a recipe that has not been inserted into a VPlan
+ // in a debugger.
+ const VPRecipeBase *DefR = V->getDefiningRecipe();
+ assert((!DefR || !DefR->getParent() || !DefR->getParent()->getPlan()) &&
+ "VPValue defined by a recipe in a VPlan?");
+
+ // Use the underlying value's name, if there is one.
+ if (auto *UV = V->getUnderlyingValue()) {
+ std::string Name;
+ raw_string_ostream S(Name);
+ UV->printAsOperand(S, false);
+ return (Twine("ir<%") + Name + ">").str();
}
- assert(
- !V->getUnderlyingValue() &&
- "Must not have assigned names for VPValues without an underlying value");
- unsigned Slot = getSlot(V);
- if (Slot == unsigned(-1))
- return "<badref>";
- else
- return (Twine("vp<%") + std::to_string(Slot) + ">").str();
+ return "<badref>";
}
bool vputils::onlyFirstLaneUsed(const VPValue *Def) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 15094c6a60004f..cc4bb7bc10fd1e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -444,27 +444,29 @@ class VPDef {
class VPlan;
class VPBasicBlock;
-/// This class can be used to assign consecutive numbers to VPValues in a
-/// VPlan without underlying values and deduplicated names to VPValues with
-/// underlying values. Allows querying the numbering and deduplicated names for
-/// printing, similar to the ModuleSlotTracker for IR values.
+/// This class can be used to assign names to VPValues. For VPValues without
+/// underlying value, assign consecutive numbers and use those as names (wrapped
+/// in vp<>). Otherwise, use the name from the underlying value (wrapped in
+/// ir<>), apending a .V version number if there are multiple uses of the same
+/// name. Allows querying names for VPValues for printing, similar to the
+/// ModuleSlotTracker for IR values.
class VPSlotTracker {
- /// Keep track of de-duplicated names assigned to VPValues with underlying IR
+ /// Keep track of versioned names assigned to VPValues with underlying IR
/// values
- DenseMap<const VPValue *, std::string> AssignedNames;
+ DenseMap<const VPValue *, std::string> VPValue2Name;
/// Keep track of the next number to use to deduplicate the base name.
- StringMap<unsigned> NameUseCount;
+ StringMap<unsigned> BaseName2Version;
- DenseMap<const VPValue *, unsigned> Slots;
+ /// Number to assign to the next VPValue without underlying value.
unsigned NextSlot = 0;
void assignSlotOrName(const VPValue *V);
void assignSlotsOrNames(const VPlan &Plan);
void assignSlotsOrNames(const VPBasicBlock *VPBB);
- /// Create a deduplicated version of \p Name for \p V by appending ".Number"
+ /// Create a versioned variant of \p Name for \p V by appending ".Number"
/// to \p Name if there are multiple uses of that name.
- void deduplicateName(const VPValue *V, StringRef Name);
+ void versionName(const VPValue *V, StringRef Name);
public:
VPSlotTracker(const VPlan *Plan = nullptr) {
@@ -472,14 +474,10 @@ class VPSlotTracker {
assignSlotsOrNames(*Plan);
}
- unsigned getSlot(const VPValue *V) const {
- auto I = Slots.find(V);
- if (I == Slots.end())
- return -1;
- return I->second;
- }
-
- std::string getName(const VPValue *) const;
+ /// Returns the name assigned to \p V, if there is one, otherwise try to
+ /// construct one from the underlying value, if there's one; else return
+ /// <badref>.
+ std::string getName(const VPValue *V) const;
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 06bd2e21470328..108b78a70fa1a9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.1> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep.1>
+; CHECK-NEXT: REPLICATE ir<%gep>.1 = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep>.1
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -1129,8 +1129,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%ptr.iv.next.1> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
-; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next.1>
+; CHECK-NEXT: REPLICATE ir<%ptr.iv.next>.1 = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
+; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next>.1
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
More information about the llvm-commits
mailing list