[llvm] [VPlan] Track VPValue names in VPlan. (PR #81411)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 11 05:42:24 PST 2024
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/81411
This patch restructures the way names for printing VPValues are handled. It updates VPlan to track assigned names for VPValues (similar to how LLVMContext tracks names for LLVM IR Values). Tracking names in a single place allows to deduplicate names and ensure all names are unique.
Now there are the following cases:
* If a VPValue has a defining recipe, its name is determined by checking its associated VPlan's mapping. The name is wrapped in vp<>. If no VPlan is associated, <badref> is used.
* If a VPValue is a live-in with an underlying IR value, the name of the IR value is used and wrapped in ir<>.
* If none of the above applies, the number assigned by VPSlotTracker is used wrapped in vp<> if an associated VPlan is provided or <badref> otherwise.
Note that contrary to LLMV's Value which always can retrieve LLVMContext, this is not possible for VPValues without defining recipe, so in some cases, the associated VPlan needs to be provided explicitly.
The new handling subsumes VPInstruction's manual handling of its name.
>From 5c96c8d51863fb6c2b96e06ffcd7b90280bb20ca Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 10 Feb 2024 21:49:22 +0000
Subject: [PATCH] [VPlan] Track VPValue names in VPlan.
This patch restructures the way names for printing VPValues are handled.
It updates VPlan to track assigned names for VPValues (similar to how
LLVMContext tracks names for LLVM IR Values). Tracking names in a single
place allows to deduplicate names and ensure all names are unique.
Now there are the following cases:
* If a VPValue has a defining recipe, its name is determined by
checking its associated VPlan's mapping. The name is wrapped in vp<>.
If no VPlan is associated, <badref> is used.
* If a VPValue is a live-in with an underlying IR value, the name of
the IR value is used and wrapped in ir<>.
* If none of the above applies, the number assigned by VPSlotTracker is
used wrapped in vp<> if an associated VPlan is provided or <badref>
otherwise.
Note that contrary to LLMV's Value which always can retrieve
LLVMContext, this is not possible for VPValues without defining recipe,
so in some cases, the associated VPlan needs to be provided explicitly.
The new handling subsumes VPInstruction's manual handling of its name.
---
.../Vectorize/LoopVectorizationPlanner.h | 23 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 13 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 82 +++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 66 ++++-
.../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 9 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 16 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 20 +-
.../AArch64/sve-tail-folding-forced.ll | 14 +-
.../LoopVectorize/AArch64/sve-widen-gep.ll | 20 +-
.../AArch64/synthesize-mask-for-call.ll | 84 +++---
.../widen-call-with-intrinsic-or-libfunc.ll | 28 +-
.../RISCV/riscv-vector-reverse.ll | 36 +--
.../first-order-recurrence-chains-vplan.ll | 46 ++--
...-order-recurrence-sink-replicate-region.ll | 132 ++++-----
.../Transforms/LoopVectorize/icmp-uniforms.ll | 10 +-
.../interleave-and-scalarize-only.ll | 20 +-
.../LoopVectorize/vplan-dot-printing.ll | 14 +-
.../LoopVectorize/vplan-iv-transforms.ll | 6 +-
.../vplan-printing-outer-loop.ll | 26 +-
.../LoopVectorize/vplan-printing.ll | 252 +++++++++---------
.../vplan-sink-scalars-and-merge-vf1.ll | 8 +-
.../vplan-sink-scalars-and-merge.ll | 250 ++++++++---------
.../vplan-unused-interleave-group.ll | 6 +-
24 files changed, 662 insertions(+), 531 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index a7ebf78e54ceb6..5aa4892bf0ffe5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -47,16 +47,19 @@ class VPBuilder {
VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
/// Insert \p VPI in BB at InsertPt if BB is set.
- VPInstruction *tryInsertInstruction(VPInstruction *VPI) {
- if (BB)
+ VPInstruction *tryInsertInstruction(VPInstruction *VPI,
+ const Twine &Name = "") {
+ if (BB) {
BB->insert(VPI, InsertPt);
+ VPI->setName(Name);
+ }
return VPI;
}
VPInstruction *createInstruction(unsigned Opcode,
ArrayRef<VPValue *> Operands, DebugLoc DL,
const Twine &Name = "") {
- return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
+ return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL), Name);
}
VPInstruction *createInstruction(unsigned Opcode,
@@ -150,7 +153,7 @@ class VPBuilder {
VPRecipeWithIRFlags::WrapFlagsTy WrapFlags,
DebugLoc DL = {}, const Twine &Name = "") {
return tryInsertInstruction(
- new VPInstruction(Opcode, Operands, WrapFlags, DL, Name));
+ new VPInstruction(Opcode, Operands, WrapFlags, DL), Name);
}
VPValue *createNot(VPValue *Operand, DebugLoc DL = {},
const Twine &Name = "") {
@@ -170,12 +173,12 @@ class VPBuilder {
VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
DebugLoc DL = {}, const Twine &Name = "",
std::optional<FastMathFlags> FMFs = std::nullopt) {
- auto *Select =
- FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
- *FMFs, DL, Name)
- : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
- DL, Name);
- return tryInsertInstruction(Select);
+ auto *Select = FMFs
+ ? new VPInstruction(Instruction::Select,
+ {Cond, TrueVal, FalseVal}, *FMFs, DL)
+ : new VPInstruction(Instruction::Select,
+ {Cond, TrueVal, FalseVal}, DL);
+ return tryInsertInstruction(Select, Name);
}
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1a7b301c35f2b8..d08aa8fcb12c24 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7262,7 +7262,7 @@ VPValue *VPBuilder::createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
return tryInsertInstruction(
- new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name));
+ new VPInstruction(Instruction::ICmp, Pred, A, B, DL), Name);
}
// This function will select a scalable VF if the target supports scalable
@@ -8610,11 +8610,11 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
// IV by VF * UF.
auto *CanonicalIVIncrement =
new VPInstruction(Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()},
- {HasNUW, false}, DL, "index.next");
+ {HasNUW, false}, DL);
CanonicalIVPHI->addOperand(CanonicalIVIncrement);
VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
- EB->appendRecipe(CanonicalIVIncrement);
+ EB->appendRecipe(CanonicalIVIncrement, "index.next");
// Add the BranchOnCount VPInstruction to the latch.
VPInstruction *BranchBack =
@@ -8781,8 +8781,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
CM.foldTailByMasking() || isa<TruncInst>(Instr)) &&
"unexpected recipe needs moving");
Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
+ Recipe->getVPSingleValue()->setName(Instr->getName());
} else
- VPBB->appendRecipe(Recipe);
+ VPBB->appendRecipe(Recipe, Instr->getName());
}
VPBlockUtils::insertBlockAfter(new VPBasicBlock(), VPBB);
@@ -8842,6 +8843,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
J++;
}
MemberR->eraseFromParent();
+ if (!Member->getType()->isVoidTy())
+ VPIG->getVPValue(J - 1)->setName(Member->getName());
}
}
@@ -9097,7 +9100,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
// ensure that it comes after all of it's inputs, including CondOp.
// Note that this transformation may leave over dead recipes (including
// CurrentLink), which will be cleaned by a later VPlan transform.
- LinkVPBB->appendRecipe(RedRecipe);
+ LinkVPBB->appendRecipe(RedRecipe, CurrentLinkI->getName());
CurrentLink->replaceAllUsesWith(RedRecipe);
PreviousLink = RedRecipe;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2c0daa82afa59f..bb603653f37ddc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -588,6 +588,15 @@ bool VPBasicBlock::isExiting() const {
return getParent()->getExitingBasicBlock() == this;
}
+void VPBasicBlock::appendRecipe(VPRecipeBase *Recipe, const Twine &Name) {
+ insert(Recipe, end());
+ if (Recipe->getNumDefinedValues() == 1)
+ Recipe->getVPSingleValue()->setName(Name);
+ else
+ assert(Name.str().empty() && "Provided a name for a recipe that does not "
+ "define exactly a single VPValue");
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const {
if (getSuccessors().empty()) {
@@ -1053,8 +1062,10 @@ static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry,
assert(OldR.getNumDefinedValues() == NewR.getNumDefinedValues() &&
"recipes must define the same number of operands");
for (const auto &[OldV, NewV] :
- zip(OldR.definedValues(), NewR.definedValues()))
+ zip(OldR.definedValues(), NewR.definedValues())) {
Old2NewVPValues[OldV] = NewV;
+ NewV->setName(OldV->getName());
+ }
}
}
@@ -1112,6 +1123,26 @@ VPlan *VPlan::duplicate() {
return NewPlan;
}
+void VPlan::setName(const VPValue *V, const Twine &Name) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ std::string N = Name.str();
+ if (N.empty())
+ return;
+ assert(!VPValue2Name.contains(V));
+ std::string CurrName = N;
+
+ if (UsedNames.contains(N))
+ CurrName = N + ".1";
+ unsigned Cnt = 2;
+ while (UsedNames.contains(CurrName)) {
+ CurrName = N + "." + std::to_string(Cnt);
+ Cnt += 1;
+ }
+ VPValue2Name[V] = CurrName;
+ UsedNames.insert(CurrName);
+#endif
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1298,18 +1329,12 @@ void VPValue::replaceUsesWithIf(
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const {
- if (const Value *UV = getUnderlyingValue()) {
+ if (!getDefiningRecipe() && getUnderlyingValue()) {
OS << "ir<";
- UV->printAsOperand(OS, false);
+ getUnderlyingValue()->printAsOperand(OS, false);
OS << ">";
- return;
- }
-
- unsigned Slot = Tracker.getSlot(this);
- if (Slot == unsigned(-1))
- OS << "<badref>";
- else
- OS << "vp<%" << Tracker.getSlot(this) << ">";
+ } else
+ OS << Tracker.getName(this);
}
void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
@@ -1319,6 +1344,31 @@ void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
}
#endif
+std::string VPValue::getName() {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ if (auto *Def = getDefiningRecipe())
+ return Def->getParent()->getPlan()->getName(this);
+ assert(isLiveIn() && "called getName() on unsupported VPValue");
+ return getLiveInIRValue()->getName().str();
+#else
+ return "";
+#endif
+}
+
+void VPValue::setName(const Twine &Name) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ auto *Def = getDefiningRecipe();
+ Def->getParent()->getPlan()->setName(this, Name);
+#endif
+}
+
+void VPValue::takeName(const VPValue *Src) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ auto *Def = getDefiningRecipe();
+ Def->getParent()->getPlan()->takeName(Src, this);
+#endif
+}
+
void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
Old2NewTy &Old2New,
InterleavedAccessInfo &IAI) {
@@ -1397,6 +1447,16 @@ void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
assignSlot(Def);
}
+std::string VPSlotTracker::getName(const VPValue *V) const {
+ if (!Plan)
+ return "<badref>";
+ std::string N = Plan->getName(V);
+ if (!N.empty())
+ return (Twine("vp<%") + N + ">").str();
+
+ return (Twine("vp<%") + std::to_string(getSlot(V)) + ">").str();
+}
+
bool vputils::onlyFirstLaneUsed(const VPValue *Def) {
return all_of(Def->users(),
[Def](const VPUser *U) { return U->onlyFirstLaneUsed(Def); });
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 162a3c4b195e53..08c0041f23ae07 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -32,6 +32,7 @@
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/ilist.h"
#include "llvm/ADT/ilist_node.h"
@@ -1162,9 +1163,6 @@ class VPInstruction : public VPRecipeWithIRFlags {
typedef unsigned char OpcodeTy;
OpcodeTy Opcode;
- /// An optional name that can be used for the generated IR instruction.
- const std::string Name;
-
/// Utility method serving execute(): generates a single instance of the
/// modeled instruction. \returns the generated value for \p Part.
/// In some cases an existing value is returned rather than a generated
@@ -1181,31 +1179,30 @@ class VPInstruction : public VPRecipeWithIRFlags {
void setUnderlyingInstr(Instruction *I) { setUnderlyingValue(I); }
public:
- VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
- const Twine &Name = "")
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
- Opcode(Opcode), Name(Name.str()) {}
+ Opcode(Opcode) {}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- DebugLoc DL = {}, const Twine &Name = "")
- : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}
+ DebugLoc DL = {})
+ : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL) {}
VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
- VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
+ VPValue *B, DebugLoc DL = {});
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "")
+ WrapFlagsTy WrapFlags, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL),
- Opcode(Opcode), Name(Name.str()) {}
+ Opcode(Opcode) {}
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
+ FastMathFlags FMFs, DebugLoc DL = {});
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
VPRecipeBase *clone() override {
SmallVector<VPValue *, 2> Operands(operands());
- auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name);
+ auto *New = new VPInstruction(Opcode, Operands, getDebugLoc());
New->transferFlags(*this);
return New;
}
@@ -2653,7 +2650,7 @@ class VPBasicBlock : public VPBlockBase {
/// Augment the existing recipes of a VPBasicBlock with an additional
/// \p Recipe as the last recipe.
- void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
+ void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = "");
/// The method which generates the output IR instructions that correspond to
/// this VPBasicBlock, thereby "executing" the VPlan.
@@ -2876,6 +2873,11 @@ class VPlan {
/// been modeled in VPlan directly.
DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ DenseMap<const VPValue *, std::string> VPValue2Name;
+ StringSet<> UsedNames;
+#endif
+
public:
/// Construct a VPlan with original preheader \p Preheader, trip count \p TC
/// and \p Entry to the plan. At the moment, \p Preheader and \p Entry need to
@@ -3016,6 +3018,42 @@ class VPlan {
LLVM_DUMP_METHOD void dump() const;
#endif
+ /// Set the name for \p V to \p Name if it is not empty. \p V must not have a
+ /// name assigned already.
+ void setName(const VPValue *V, const Twine &Name);
+
+ /// Remove the assigned name for \p V, if there is one.
+ void removeName(const VPValue *V) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ if (VPValue2Name.contains(V)) {
+ UsedNames.erase(VPValue2Name[V]);
+ VPValue2Name.erase(V);
+ }
+#endif
+ }
+
+ /// Take the name for \p Src and move it to \p Dst. The name of \p Src will be
+ /// empty afterwards.
+ void takeName(const VPValue *Src, const VPValue *Dst) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ std::string N = VPValue2Name.lookup(Src);
+ if (N.empty())
+ return;
+ VPValue2Name[Dst] = N;
+ VPValue2Name.erase(Src);
+#endif
+ }
+
+ /// Return the name assigned to \p V or an empty string if no name has been
+ /// assigned.
+ std::string getName(const VPValue *V) const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ return VPValue2Name.lookup(V);
+#else
+ return "";
+#endif
+ }
+
/// Returns a range mapping the values the range \p Operands to their
/// corresponding VPValues.
iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 94456bf858d9c5..f5b34a740a21b0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -297,7 +297,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
if (Br->isConditional()) {
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
VPBB->appendRecipe(
- new VPInstruction(VPInstruction::BranchOnCond, {Cond}));
+ new VPInstruction(VPInstruction::BranchOnCond, {Cond}),
+ Inst->getName());
}
// Skip the rest of the Instruction processing for Branch instructions.
@@ -310,7 +311,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// an empty VPInstruction that we will fix once the whole plain CFG has
// been built.
NewVPV = new VPWidenPHIRecipe(Phi);
- VPBB->appendRecipe(cast<VPWidenPHIRecipe>(NewVPV));
+ VPBB->appendRecipe(cast<VPWidenPHIRecipe>(NewVPV), Inst->getName());
PhisToFix.push_back(Phi);
} else {
// Translate LLVM-IR operands into VPValue operands and set them in the
@@ -321,8 +322,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// Build VPInstruction for any arbitrary Instruction without specific
// representation in VPlan.
- NewVPV = cast<VPInstruction>(
- VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
+ NewVPV = cast<VPInstruction>(VPIRBuilder.createNaryOp(
+ Inst->getOpcode(), VPOperands, Inst, Inst->getName()));
}
IRDef2VPValue[Inst] = NewVPV;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9ee0cb2bd61530..c24e0d72c222f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -226,6 +226,8 @@ void VPRecipeBase::removeFromParent() {
iplist<VPRecipeBase>::iterator VPRecipeBase::eraseFromParent() {
assert(getParent() && "Recipe not in any VPBasicBlock");
+ for (auto *V : definedValues())
+ getParent()->getPlan()->removeName(V);
return getParent()->getRecipeList().erase(getIterator());
}
@@ -255,20 +257,19 @@ FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const {
}
VPInstruction::VPInstruction(unsigned Opcode, CmpInst::Predicate Pred,
- VPValue *A, VPValue *B, DebugLoc DL,
- const Twine &Name)
+ VPValue *A, VPValue *B, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef<VPValue *>({A, B}),
Pred, DL),
- Opcode(Opcode), Name(Name.str()) {
+ Opcode(Opcode) {
assert(Opcode == Instruction::ICmp &&
"only ICmp predicates supported at the moment");
}
VPInstruction::VPInstruction(unsigned Opcode,
std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL, const Twine &Name)
+ FastMathFlags FMFs, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL),
- Opcode(Opcode), Name(Name.str()) {
+ Opcode(Opcode) {
// Make sure the VPInstruction is a floating-point operation.
assert(isFPMathOp() && "this op can't take fast-math flags");
}
@@ -277,6 +278,7 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
unsigned Part) {
IRBuilderBase &Builder = State.Builder;
Builder.SetCurrentDebugLocation(getDebugLoc());
+ std::string Name = getParent()->getPlan()->getName(this);
if (Instruction::isBinaryOp(getOpcode())) {
if (Part != 0 && vputils::onlyFirstPartUsed(this))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 71f5285f90236b..bb1db219704439 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -88,9 +88,10 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
}
NewRecipe->insertBefore(&Ingredient);
- if (NewRecipe->getNumDefinedValues() == 1)
+ if (NewRecipe->getNumDefinedValues() == 1) {
VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
- else
+ NewRecipe->getVPSingleValue()->takeName(VPV);
+ } else
assert(NewRecipe->getNumDefinedValues() == 0 &&
"Only recpies with zero or one defined values expected");
Ingredient.eraseFromParent();
@@ -163,6 +164,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
// TODO: add ".cloned" suffix to name of Clone's VPValue.
Clone->insertBefore(SinkCandidate);
+ if (!I->getName().empty())
+ Clone->setName(I->getName() + ".cloned");
SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) {
return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
});
@@ -318,6 +321,8 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
PHIRecipe->setOperand(0, RecipeWithoutMask);
}
PredRecipe->eraseFromParent();
+ Plan.setName(RecipeWithoutMask,
+ RecipeWithoutMask->getUnderlyingValue()->getName());
auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true);
@@ -1177,10 +1182,11 @@ void VPlanTransforms::addActiveLaneMask(
LaneMask = addVPLaneMaskPhiAndUpdateExitBranch(
Plan, DataAndControlFlowWithoutRuntimeCheck);
} else {
- LaneMask = new VPInstruction(VPInstruction::ActiveLaneMask,
- {WideCanonicalIV, Plan.getTripCount()},
- nullptr, "active.lane.mask");
+ LaneMask =
+ new VPInstruction(VPInstruction::ActiveLaneMask,
+ {WideCanonicalIV, Plan.getTripCount()}, nullptr);
LaneMask->insertAfter(WideCanonicalIV);
+ LaneMask->setName("active.lane.mask");
}
// Walk users of WideCanonicalIV and replace all compares of the form
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index c85f7715feaa2a..e7ddc472484481 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -192,6 +192,21 @@ class VPValue {
/// is a live-in value.
/// TODO: Also handle recipes defined in pre-header blocks.
bool isDefinedOutsideVectorRegions() const { return !hasDefiningRecipe(); }
+
+ /// Returns the name associated with the VPValue. In case the VPValue has a
+ /// defining recipe, look up the name in the containing VPlan. In case it is a
+ /// live-in with an underlying IR value, return the name of the underlying
+ /// value. Calling getName() in other cases is invalid.
+ std::string getName();
+
+ /// Sets the name for this VPValue to \p Name. The VPValue must have a
+ /// defining recipe inserted in a VPlan.
+ void setName(const Twine &Name);
+
+ /// Take the name for \p Src and move it to this VPValue. The name of \p Src
+ /// will be empty afterwards. The VPValue must have a defining recipe inserted
+ /// in a VPlan.
+ void takeName(const VPValue *Src);
};
typedef DenseMap<Value *, VPValue *> Value2VPValueTy;
@@ -446,6 +461,7 @@ class VPBasicBlock;
/// VPlan and allows querying the numbering for printing, similar to the
/// ModuleSlotTracker for IR values.
class VPSlotTracker {
+ const VPlan *Plan;
DenseMap<const VPValue *, unsigned> Slots;
unsigned NextSlot = 0;
@@ -454,7 +470,7 @@ class VPSlotTracker {
void assignSlots(const VPBasicBlock *VPBB);
public:
- VPSlotTracker(const VPlan *Plan = nullptr) {
+ VPSlotTracker(const VPlan *Plan = nullptr) : Plan(Plan) {
if (Plan)
assignSlots(*Plan);
}
@@ -465,6 +481,8 @@ class VPSlotTracker {
return -1;
return I->second;
}
+
+ std::string getName(const VPValue *V) const;
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
index cc72dfa4ce639f..33708bad471da3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
@@ -19,20 +19,20 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]>
-; VPLANS-NEXT: EMIT vp<[[VF:%[0-9]+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%[0-9]+]]> = active lane mask vp<[[VF]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
+; VPLANS-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, vp<[[TC]]>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
; VPLANS-NEXT: <x1> vector loop: {
; VPLANS-NEXT: vector.body:
; VPLANS-NEXT: EMIT vp<[[INDV:%[0-9]+]]> = CANONICAL-INDUCTION
-; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%[0-9]+]]>
+; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<[[LANEMASK_LOOP:%.+]]>
; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1>
-; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
-; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer ir<%gep>
+; VPLANS-NEXT: CLONE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
+; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer vp<%gep>
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
-; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
-; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]>
+; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
+; VPLANS-NEXT: EMIT vp<[[INC:%.+]]> = VF * Part + vp<[[INDV]]>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[NEWTC]]>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index 899fcce5c02aba..eb7e5997b96c87 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -19,16 +19,16 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: EMIT ir<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8
-; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
-; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ptr.iv.1>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR2]]>
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1>
-; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer ir<%ptr.iv.2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, ir<%add>
+; CHECK-NEXT: EMIT vp<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8
+; CHECK-NEXT: EMIT vp<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
+; CHECK-NEXT: WIDEN-GEP Var[Inv] vp<%ptr.iv.2.next> = getelementptr inbounds vp<%ptr.iv.2>, ir<1>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ptr.iv.1>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%ptr.iv.2.next>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%ptr.iv.2>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR2]]>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<%lv>, ir<1>
+; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer vp<%ptr.iv.2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index ad0d40ee7fbde9..02e348a8e6f57b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -21,13 +21,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REPLICATE vp<%call> = call @foo(vp<%load>)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -50,13 +50,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -84,13 +84,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -113,13 +113,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir<true>) (using library function: foo_vector_fixed4_mask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>, ir<true>) (using library function: foo_vector_fixed4_mask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -146,13 +146,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -175,13 +175,13 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask)
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
index f67b932b04a0d6..9b8cbb37ea2de7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
@@ -19,13 +19,13 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double
-; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64)
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst>
+; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double
+; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using library function: __simd_sin_v2f64)
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -48,13 +48,13 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double
-; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic)
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst>
+; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double
+; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using vector intrinsic)
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 1bcd7a2e009e0b..ec70c0e3657fd0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -64,15 +64,15 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
+; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0>
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%add9> = add vp<[[L]]>, ir<1>
+; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add9>
; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -204,15 +204,15 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
-; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1>
+; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0>
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%conv1> = fadd vp<[[L]]>, ir<1.000000e+00>
+; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%conv1>
; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index 15e77f3a48470c..db85e9d8a34c9a 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -15,17 +15,17 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next>
-; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]>
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
+; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next>
+; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -68,20 +68,20 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next>
-; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]>
-; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice ir<%for.3>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
-; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add.2>
+; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next>
+; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]>
+; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice vp<%for.3>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: WIDEN vp<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]>
+; CHECK-NEXT: WIDEN vp<%add.2> = add vp<%add.1>, vp<[[FOR3_SPLICE]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add.2>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 833d55f09294e8..06f7a7be352f4b 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -20,10 +20,10 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -32,19 +32,19 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED1]]> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED1]]> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -53,14 +53,14 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%conv>, vp<%rem>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -111,11 +111,11 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -124,15 +124,15 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%rem>, vp<%recur.next>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
@@ -181,12 +181,12 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%and.red> = phi ir<1234>, vp<%and.red.next>
; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
; CHECK-NEXT: Successor(s): pred.srem
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.srem: {
@@ -195,19 +195,19 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
; CHECK-NEXT: Successor(s): pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next>
-; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add>
-; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, vp<%recur.next>
+; CHECK-NEXT: WIDEN vp<%and.red.next> = and vp<%and.red>, vp<%add>
+; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, vp<%and.red.next>, vp<%and.red>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -215,7 +215,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%and.red>, vp<[[SEL]]>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]>
@@ -257,11 +257,11 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -270,18 +270,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
+; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED]]> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED]]> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK: <xVFxUF> pred.store: {
@@ -290,18 +290,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep>
-; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2>
-; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%lv.2> = load vp<%gep>
+; CHECK-NEXT: REPLICATE vp<%conv.lv.2> = sext vp<%lv.2>
+; CHECK-NEXT: REPLICATE vp<%add.1> = add vp<%conv>, vp<%rem>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%add.1>, vp<%conv.lv.2>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%rem>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.2>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
@@ -360,11 +360,11 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -374,17 +374,17 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
-; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep>
-; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2>
+; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%rem.div> = sdiv ir<20>, vp<%rem>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep.2>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem.div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem.div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
@@ -435,12 +435,12 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%.pn> = phi ir<0>, vp<[[L:%.+]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]>, vp<[[BTC]]>
-; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src>
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn>, ir<[[L]]>
+; CHECK-NEXT: CLONE vp<[[L]]> = load ir<%src>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%.pn>, vp<[[L]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -449,14 +449,14 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT: REPLICATE vp<%val> = sdiv vp<[[SPLICE]]>, ir<%x>
; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%val>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = vp<%val>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index b4ab6f7e8ceb70..f0b8ded142210d 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -49,9 +49,9 @@ for.end:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13>
-; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>
+; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%cond0> = icmp ult vp<%iv>, ir<13>
+; CHECK-NEXT: WIDEN-SELECT vp<%s> = select vp<%cond0>, ir<10>, ir<20>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -61,8 +61,8 @@ for.end:
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index 81cc2024bb31a5..54a66dd424faed 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -21,9 +21,9 @@
; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%start> + vp<[[CAN_IV]]> * ir<1>
; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>)
-; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]>
-; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx>
+; DBG-NEXT: CLONE vp<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>)
+; DBG-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]>
+; DBG-NEXT: CLONE store vp<%min>, vp<%arrayidx>
; DBG-NEXT: EMIT vp<[[INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; DBG-NEXT: EMIT branch-on-count vp<[[INC]]>, vp<[[VEC_TC]]>
; DBG-NEXT: No successors
@@ -89,14 +89,14 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-EMPTY:
; DBG-NEXT: pred.store.if:
; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; DBG-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]>
-; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src>
-; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]>
-; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst>
+; DBG-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]>
+; DBG-NEXT: CLONE vp<%l> = load vp<%gep.src>
+; DBG-NEXT: CLONE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]>
+; DBG-NEXT: CLONE store vp<%l>, vp<%gep.dst>
; DBG-NEXT: Successor(s): pred.store.continue
; DBG-EMPTY:
; DBG-NEXT: pred.store.continue:
-; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l>
+; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = vp<%l>
; DBG-NEXT: No successors
; DBG-NEXT: }
; DBG-NEXT: Successor(s): cond.false.1
@@ -190,10 +190,10 @@ exit:
; DBG-NEXT: <x1> vector loop: {
; DBG-NEXT: vector.body:
; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]>
+; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]>
; DBG-NEXT: SCALAR-CAST vp<[[TRUNC_IV:%.+]]> = trunc vp<[[CAN_IV]]> to i32
; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[TRUNC_IV]]>, vp<[[CAST]]>
-; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[SCALAR_STEPS]]>
+; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[SCALAR_STEPS]]>
; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst>
; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VTC]]>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index 12b8e657aabf12..85d035f1ddaee0 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -28,13 +28,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: "vector.body:\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION ir\<0\>, vp\<[[CAN_IV_NEXT:%.+]]\>\l" +
; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" +
-; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer ir\<%arrayidx\>\l" +
-; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
-; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" +
-; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
-; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer ir\<%arrayidx2\>\l" +
-; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" +
+; CHECK-NEXT: " CLONE vp\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer vp\<%arrayidx\>\l" +
+; CHECK-NEXT: " WIDEN vp\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
+; CHECK-NEXT: " WIDEN-CALL vp\<%call\> = call @llvm.sqrt.f32(vp\<%lv\>) (using vector intrinsic)\l" +
+; CHECK-NEXT: " CLONE vp\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
+; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer vp\<%arrayidx2\>\l" +
+; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, vp\<%call\>\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" +
; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" +
; CHECK-NEXT: "No successors\l"
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 7ab2459ada2ed6..00996e7eee5664 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -16,9 +16,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next.p, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%iv>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%iv>
; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 65df5973ac75b6..1795c6a4fe33f5 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -15,28 +15,28 @@ define void @foo(i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
-; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
-; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1>
-; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
+; CHECK-NEXT: WIDEN-PHI vp<%outer.iv> = phi ir<0>, vp<%outer.iv.next>
+; CHECK-NEXT: EMIT vp<%gep.1> = getelementptr ir<@arr2>, ir<0>, vp<%outer.iv>
+; CHECK-NEXT: EMIT store vp<%outer.iv>, vp<%gep.1>
+; CHECK-NEXT: EMIT vp<%add> = add vp<%outer.iv>, ir<%n>
; CHECK-NEXT: Successor(s): inner
; CHECK-EMPTY:
; CHECK-NEXT: <x1> inner: {
; CHECK-NEXT: inner:
-; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
-; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
-; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2>
-; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
+; CHECK-NEXT: WIDEN-PHI vp<%inner.iv> = phi ir<0>, vp<%inner.iv.next>
+; CHECK-NEXT: EMIT vp<%gep.2> = getelementptr ir<@arr>, ir<0>, vp<%inner.iv>, vp<%outer.iv>
+; CHECK-NEXT: EMIT store vp<%add>, vp<%gep.2>
+; CHECK-NEXT: EMIT vp<%inner.iv.next> = add vp<%inner.iv>, ir<1>
+; CHECK-NEXT: EMIT vp<%inner.ec> = icmp vp<%inner.iv.next>, ir<8>
+; CHECK-NEXT: EMIT branch-on-cond vp<%inner.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): outer.latch
; CHECK-EMPTY:
; CHECK-NEXT: outer.latch:
-; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
+; CHECK-NEXT: EMIT vp<%outer.iv.next> = add vp<%outer.iv>, ir<1>
+; CHECK-NEXT: EMIT vp<%outer.ec> = icmp vp<%outer.iv.next>, ir<8>
+; CHECK-NEXT: EMIT branch-on-cond vp<%outer.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 89178953010fe0..372e99583c4066 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -20,13 +20,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call>
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CALL vp<%call> = call @llvm.sqrt.f32(vp<%lv>)
+; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -71,15 +71,15 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr inbounds ir<%y>, ir<%iv>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z>
-; CHECK-NEXT: WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
-; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
+; CHECK-NEXT: WIDEN-GEP Inv[Var] vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<%iv>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%cmp> = icmp eq vp<%arrayidx>, ir<%z>
+; CHECK-NEXT: WIDEN-SELECT vp<%sel> = select vp<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
+; CHECK-NEXT: WIDEN vp<%add> = fadd vp<%lv>, vp<%sel>
+; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -124,12 +124,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -137,7 +137,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%red>, ir<%red.next>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out float %red.next.lcssa = vp<[[RED_RES]]>
@@ -173,12 +173,12 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1>
; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>) (with final reduction value stored in invariant address sank outside of loop)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -186,7 +186,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result ir<%red>, ir<%red.next>
+; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
@@ -227,30 +227,30 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5>
+; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%i>, ir<5>
; CHECK-NEXT: Successor(s): pred.udiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.udiv: {
; CHECK-NEXT: pred.udiv.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp>
+; CHECK-NEXT: BRANCH-ON-MASK vp<%cmp>
; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.if:
-; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
+; CHECK-NEXT: REPLICATE vp<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
; CHECK-NEXT: Successor(s): pred.udiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%tmp4>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%tmp4>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%cmp>
-; CHECK-NEXT: BLEND ir<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp>
-; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%d>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%cmp>
+; CHECK-NEXT: BLEND vp<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/vp<%cmp>
+; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%d>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -303,19 +303,19 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4>
-; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0>
-; CHECK-NEXT: ir<%AB.0> = load from index 0
-; CHECK-NEXT: ir<%AB.1> = load from index 1
-; CHECK-NEXT: ir<%AB.3> = load from index 3
-; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<[[STEPS]]>, ir<3>
-; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%AB.0>, ir<%AB.1>
-; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, ir<%iv.plus.3>
-; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%gep.CD.3>
-; CHECK-NEXT: store ir<%add> to index 0
+; CHECK-NEXT: CLONE vp<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, vp<%gep.AB.0>
+; CHECK-NEXT: vp<%AB.0> = load from index 0
+; CHECK-NEXT: vp<%AB.1> = load from index 1
+; CHECK-NEXT: vp<%AB.3> = load from index 3
+; CHECK-NEXT: CLONE vp<%iv.plus.3> = add vp<[[STEPS]]>, ir<3>
+; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%AB.0>, vp<%AB.1>
+; CHECK-NEXT: CLONE vp<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, vp<%iv.plus.3>
+; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at <badref>, vp<%gep.CD.3>
+; CHECK-NEXT: store vp<%add> to index 0
; CHECK-NEXT: store ir<1> to index 1
; CHECK-NEXT: store ir<2> to index 2
-; CHECK-NEXT: store ir<%AB.3> to index 3
+; CHECK-NEXT: store vp<%AB.3> to index 3
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -370,16 +370,16 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%sum.07> = phi ir<0.000000e+00>, vp<%muladd.1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN ir<%l.b> = load vp<[[VEC_PTR2]]>
-; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b>
-; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
+; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx>
+; CHECK-NEXT: WIDEN vp<%l.a> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2>
+; CHECK-NEXT: WIDEN vp<%l.b> = load vp<[[VEC_PTR2]]>
+; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz vp<%l.a>, vp<%l.b>
+; CHECK-NEXT: REDUCE vp<[[MULADD:%.+]]> = vp<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -387,7 +387,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%sum.07>, ir<[[MULADD]]>
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%sum.07>, vp<[[MULADD]]>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: Live-out float %muladd.lcssa = vp<[[RED_RES]]>
@@ -426,15 +426,15 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%isd>
-; CHECK-NEXT: WIDEN ir<%lsd> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%psd> = add nuw nsw ir<%lsd>, ir<23>
-; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100>
-; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3
-; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200>
-; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, ir<%cmp2>, ir<false>, !dbg /tmp/s.c:5:21
-; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, ir<%cmp1>
+; CHECK-NEXT: CLONE vp<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%isd>
+; CHECK-NEXT: WIDEN vp<%lsd> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%psd> = add nuw nsw vp<%lsd>, ir<23>
+; CHECK-NEXT: WIDEN vp<%cmp1> = icmp slt vp<%lsd>, ir<100>
+; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not vp<%cmp1>, !dbg /tmp/s.c:5:3
+; CHECK-NEXT: WIDEN vp<%cmp2> = icmp sge vp<%lsd>, ir<200>
+; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, vp<%cmp2>, ir<false>, !dbg /tmp/s.c:5:21
+; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, vp<%cmp1>
; CHECK-NEXT: Successor(s): pred.sdiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.sdiv: {
@@ -443,21 +443,21 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.if:
-; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V)
+; CHECK-NEXT: REPLICATE vp<%sd1> = sdiv vp<%psd>, vp<%lsd> (S->V)
; CHECK-NEXT: Successor(s): pred.sdiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = ir<%sd1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = vp<%sd1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
-; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2>
+; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp2>
; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir<false>
-; CHECK-NEXT: BLEND ir<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%isd>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%ysd.0>
+; CHECK-NEXT: BLEND vp<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> vp<%psd>/vp<[[SEL2]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%isd>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%ysd.0>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -521,12 +521,12 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION\l" +
; CHECK-NEXT: " %iv = phi %iv.next, 0\l" +
-; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]>
+; CHECK-NEXT: " vp<%v2>, vp<[[EXP_SCEV]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]>
-; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep>
+; CHECK-NEXT: WIDEN vp<%v3> = add nuw vp<%v2>, ir<1>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%v3>, vp<%gep>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
; CHECK-NEXT: No successors
@@ -572,9 +572,9 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<%iv>, ir<%off>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep>
; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<0>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
@@ -585,7 +585,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: Live-out i32 %lcssa = ir<%add>
+; CHECK-NEXT: Live-out i32 %lcssa = vp<%add>
; CHECK-NEXT: }
;
entry:
@@ -619,15 +619,15 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.y>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%add> = fadd nnan ir<%lv>, ir<1.000000e+00>
-; CHECK-NEXT: WIDEN ir<%mul> = fmul reassoc nnan ninf nsz arcp contract afn ir<%add>, ir<2.000000e+00>
-; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00>
-; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%div>
+; CHECK-NEXT: CLONE vp<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.y>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%add> = fadd nnan vp<%lv>, ir<1.000000e+00>
+; CHECK-NEXT: WIDEN vp<%mul> = fmul reassoc nnan ninf nsz arcp contract afn vp<%add>, ir<2.000000e+00>
+; CHECK-NEXT: WIDEN vp<%div> = fdiv reassoc nsz contract vp<%mul>, ir<2.000000e+00>
+; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%div>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -672,14 +672,14 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%div.1> = udiv exact ir<%lv>, ir<20>
-; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60>
-; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add>
+; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%div.1> = udiv exact vp<%lv>, ir<20>
+; CHECK-NEXT: WIDEN vp<%div.2> = udiv vp<%lv>, ir<60>
+; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%div.1>, vp<%div.2>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -723,36 +723,36 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ld.addr>
-; CHECK-NEXT: WIDEN ir<%ld.value> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%ifcond> = fcmp oeq ir<%ld.value>, ir<5.000000e+00>
+; CHECK-NEXT: CLONE vp<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ld.addr>
+; CHECK-NEXT: WIDEN vp<%ld.value> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%ifcond> = fcmp oeq vp<%ld.value>, ir<5.000000e+00>
; CHECK-NEXT: Successor(s): pred.call
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.call: {
; CHECK-NEXT: pred.call.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%ifcond>
+; CHECK-NEXT: BRANCH-ON-MASK vp<%ifcond>
; CHECK-NEXT: Successor(s): pred.call.if, pred.call.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.call.if:
-; CHECK-NEXT: REPLICATE ir<%foo.ret.1> = call nnan ninf nsz @foo(ir<%ld.value>) (S->V)
-; CHECK-NEXT: REPLICATE ir<%foo.ret.2> = call @foo(ir<%ld.value>) (S->V)
+; CHECK-NEXT: REPLICATE vp<%foo.ret.1> = call nnan ninf nsz @foo(vp<%ld.value>) (S->V)
+; CHECK-NEXT: REPLICATE vp<%foo.ret.2> = call @foo(vp<%ld.value>) (S->V)
; CHECK-NEXT: Successor(s): pred.call.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.call.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = ir<%foo.ret.1>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = ir<%foo.ret.2>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = vp<%foo.ret.1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = vp<%foo.ret.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.then.1
; CHECK-EMPTY:
; CHECK-NEXT: if.then.1:
-; CHECK-NEXT: WIDEN ir<%fadd> = fadd vp<[[PHI1]]>, vp<[[PHI2]]>
-; CHECK-NEXT: EMIT vp<[[NOT_COND:%.+]]> = not ir<%ifcond>
-; CHECK-NEXT: BLEND ir<%st.value> = ir<%ld.value>/vp<[[NOT_COND]]> ir<%fadd>/ir<%ifcond>
-; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%st.addr>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%st.value>
+; CHECK-NEXT: WIDEN vp<%fadd> = fadd vp<[[PHI1]]>, vp<[[PHI2]]>
+; CHECK-NEXT: EMIT vp<[[NOT_COND:%.+]]> = not vp<%ifcond>
+; CHECK-NEXT: BLEND vp<%st.value> = vp<%ld.value>/vp<[[NOT_COND]]> vp<%fadd>/vp<%ifcond>
+; CHECK-NEXT: CLONE vp<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%st.addr>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%st.value>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -806,14 +806,14 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%or.1> = or disjoint ir<%lv>, ir<1>
-; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3>
-; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%add>
+; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%or.1> = or disjoint vp<%lv>, ir<1>
+; CHECK-NEXT: WIDEN vp<%or.2> = or vp<%lv>, ir<3>
+; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%or.1>, vp<%or.2>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x>
+; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%add>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
@@ -857,11 +857,11 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l>
-; CHECK-NEXT: REPLICATE store ir<%zext>, ir<%p1>
+; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%p>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN-CAST vp<%zext> = zext nneg vp<%l>
+; CHECK-NEXT: REPLICATE store vp<%zext>, ir<%p1>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index 8c18c107b39fc1..7bf5e6bd76bd3d 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -19,9 +19,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]>
-; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2>
-; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2>
+; CHECK-NEXT: CLONE vp<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]>
+; CHECK-NEXT: CLONE vp<%tmp3> = load vp<%tmp2>
+; CHECK-NEXT: CLONE store ir<0>, vp<%tmp2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -30,7 +30,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
+; CHECK-NEXT: CLONE store vp<%tmp3>, vp<%tmp2>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 9b9c3e704852a1..3019a7e29e4e33 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -27,7 +27,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -37,16 +37,16 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]
-; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add>
-; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<%lv.b>, ir<10>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]
+; CHECK-NEXT: REPLICATE vp<%mul> = mul ir<2>, vp<%add>
+; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -96,7 +96,7 @@ exit:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK: <xVFxUF> pred.load: {
@@ -106,17 +106,17 @@ exit:
; CHECK: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: loop.0:
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -125,9 +125,9 @@ exit:
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE vp<%add> = add vp<[[PRED]]>, ir<10>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
@@ -180,7 +180,7 @@ exit:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK: <xVFxUF> pred.load: {
@@ -190,18 +190,18 @@ exit:
; CHECK: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: loop.0:
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<10>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add>
+; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, ir<10>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, vp<%add>
; CHECK-NEXT: Successor(s): pred.store
; CHECK: <xVFxUF> pred.store: {
@@ -210,8 +210,8 @@ exit:
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK: pred.store.continue:
@@ -265,10 +265,10 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]>, vp<[[BTC]]>
-; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0>
-; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform>
-; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k>
-; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp>
+; CHECK-NEXT: CLONE vp<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0>
+; CHECK-NEXT: CLONE vp<%lv> = load vp<%gep.A.uniform>
+; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%iv>, ir<%k>
+; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp>
; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<[[NOT2]]>, ir<false>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
@@ -279,8 +279,8 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B>
+; CHECK-NEXT: REPLICATE vp<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%lv>, vp<%gep.B>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -337,10 +337,10 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.1>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%c.1> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.1>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -350,21 +350,21 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.1>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
-; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
+; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -373,8 +373,8 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -437,11 +437,11 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
+; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: WIDEN vp<%c.1> = icmp ugt vp<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -451,22 +451,22 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
-; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
-; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]>, ir<%c.1>, ir<false>
+; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
+; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]>, vp<%c.1>, ir<false>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -475,8 +475,8 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -546,10 +546,10 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10>
+; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, vp<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -559,22 +559,22 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0>
; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
-; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
-; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]>
+; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]>, vp<%c.0>, ir<false>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -583,10 +583,10 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a>
-; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, ir<%mul>
-; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store ir<0>, vp<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<%mul>
+; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.c>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -656,7 +656,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -665,26 +665,26 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
-; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c>
-; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b>
+; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%lv.a>, vp<%gep.c>
+; CHECK-NEXT: REPLICATE store vp<%lv.b>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
; CHECK-EMPTY:
; CHECK-NEXT: loop.3:
-; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, ir<%c.0>, ir<false>
-; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>
+; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j>
+; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<%c.0>, ir<false>
+; CHECK-NEXT: WIDEN vp<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -693,8 +693,8 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1>
+; CHECK-NEXT: REPLICATE vp<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.c.1>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -760,7 +760,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -770,15 +770,15 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
-; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a>
-; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<%lv.a>, vp<%lv.a>
+; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -827,10 +827,10 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[PRED:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]>
+; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -839,17 +839,17 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = vp<%lv.a>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[PRED]]>
+; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[PRED]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -858,12 +858,12 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>
-; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>
+; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
@@ -915,15 +915,15 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
-; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
-; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
-; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
-; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>
+; CHECK-NEXT: REPLICATE vp<%l1> = load ir<%src>
+; CHECK-NEXT: REPLICATE vp<%l2> = trunc vp<%l1>
+; CHECK-NEXT: REPLICATE vp<%cmp> = icmp eq vp<%l1>, ir<0>
+; CHECK-NEXT: REPLICATE vp<%sel> = select vp<%cmp>, ir<5>, vp<%l2>
+; CHECK-NEXT: REPLICATE store vp<%sel>, ir<%dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%l1>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%l1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.then.1
@@ -973,11 +973,11 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>
-; CHECK-NEXT: WIDEN ir<%0> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00>
-; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not ir<%pred>
+; CHECK-NEXT: CLONE vp<%gep.cloned> = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.cloned>
+; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%pred> = fcmp oeq vp<[[L]]>, ir<0.000000e+00>
+; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not vp<%pred>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep>
+; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, vp<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -1055,14 +1055,14 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
-; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]>
-; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst>
+; CHECK-NEXT: REPLICATE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE vp<%l> = load vp<%gep.src>
+; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: REPLICATE store vp<%l>, vp<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = ir<%l>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = vp<%l>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.1
@@ -1113,12 +1113,12 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1
-; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0>
-; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1>
+; CHECK-NEXT: EMIT vp<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1
+; CHECK-NEXT: CLONE vp<%ptr.iv.next.cloned> = getelementptr inbounds vp<%ptr.iv>, ir<-1>
+; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%ptr.iv.next.cloned>
+; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT: WIDEN vp<%c.1> = icmp eq vp<%l>, ir<0>
+; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not vp<%c.1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -1127,8 +1127,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1>
-; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next>
+; CHECK-NEXT: REPLICATE vp<%ptr.iv.next> = getelementptr inbounds vp<%ptr.iv>, ir<-1>
+; CHECK-NEXT: REPLICATE store ir<95>, vp<%ptr.iv.next>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
index 5ea27994b356da..27d81de260d3b9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
@@ -18,9 +18,9 @@ define void @test_unused_interleave(ptr %src, i32 %length) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%3>
-; CHECK-NEXT: EMIT vp<%3> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%3>, vp<%1>
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
More information about the llvm-commits
mailing list