[llvm] 8777cb6 - [VPlan] Remove reliance on underlying instr for ScalarIVSteps (NFCI).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 2 08:23:26 PST 2022
Author: Florian Hahn
Date: 2022-03-02T16:23:19Z
New Revision: 8777cb66a82f286b9e566b89b121c5d1c403da45
URL: https://github.com/llvm/llvm-project/commit/8777cb66a82f286b9e566b89b121c5d1c403da45
DIFF: https://github.com/llvm/llvm-project/commit/8777cb66a82f286b9e566b89b121c5d1c403da45.diff
LOG: [VPlan] Remove reliance on underlying instr for ScalarIVSteps (NFCI).
Instead of relying on underlying instructions, this patch updates
VPScalarIVStepsRecipe to only store the required type information.
This removes access to unrelated information, as well as avoiding issues
with the same underlying instruction being shared by multiple recipes.
This change should only change the debug output and not cause any
codegen changes, hence NFCI.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
llvm/test/Transforms/LoopVectorize/vplan-printing.ll
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d56d6c4cd426d..810240b9527ff 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1132,6 +1132,7 @@ void InnerLoopVectorizer::collectPoisonGeneratingRecipes(
// handled.
if (isa<VPWidenMemoryInstructionRecipe>(CurRec) ||
isa<VPInterleaveRecipe>(CurRec) ||
+ isa<VPScalarIVStepsRecipe>(CurRec) ||
isa<VPCanonicalIVPHIRecipe>(CurRec))
continue;
@@ -9736,26 +9737,24 @@ void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
IndDesc.getInductionBinOp()->getFastMathFlags());
Value *Step = State.get(getStepValue(), VPIteration(0, 0));
- auto *Trunc = dyn_cast<TruncInst>(getUnderlyingValue());
auto CreateScalarIV = [&](Value *&Step) -> Value * {
Value *ScalarIV = State.get(getCanonicalIV(), VPIteration(0, 0));
auto *CanonicalIV = State.get(getParent()->getPlan()->getCanonicalIV(), 0);
- if (!isCanonical() || CanonicalIV->getType() != IV->getType()) {
- ScalarIV = IV->getType()->isIntegerTy()
- ? State.Builder.CreateSExtOrTrunc(ScalarIV, IV->getType())
- : State.Builder.CreateCast(Instruction::SIToFP, ScalarIV,
- IV->getType());
+ if (!isCanonical() || CanonicalIV->getType() != Ty) {
+ ScalarIV =
+ Ty->isIntegerTy()
+ ? State.Builder.CreateSExtOrTrunc(ScalarIV, Ty)
+ : State.Builder.CreateCast(Instruction::SIToFP, ScalarIV, Ty);
ScalarIV = emitTransformedIndex(State.Builder, ScalarIV,
getStartValue()->getLiveInIRValue(), Step,
IndDesc);
ScalarIV->setName("offset.idx");
}
- if (Trunc) {
- auto *TruncType = cast<IntegerType>(Trunc->getType());
+ if (TruncToTy) {
assert(Step->getType()->isIntegerTy() &&
"Truncation requires an integer step");
- ScalarIV = State.Builder.CreateTrunc(ScalarIV, TruncType);
- Step = State.Builder.CreateTrunc(Step, TruncType);
+ ScalarIV = State.Builder.CreateTrunc(ScalarIV, TruncToTy);
+ Step = State.Builder.CreateTrunc(Step, TruncToTy);
}
return ScalarIV;
};
@@ -9783,8 +9782,6 @@ void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
ScalarIV, State.Builder.CreateMul(StartIdx, Step), "induction");
}
State.set(this, EntryPart, Part);
- if (Trunc)
- State.ILV->addMetadata(EntryPart, Trunc);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4a2e2fcbb4f33..8ad95110ca800 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1849,15 +1849,19 @@ class VPWidenCanonicalIVRecipe : public VPRecipeBase, public VPValue {
/// A recipe for handling phi nodes of integer and floating-point inductions,
/// producing their scalar values.
class VPScalarIVStepsRecipe : public VPRecipeBase, public VPValue {
- PHINode *IV;
+ /// Scalar type to use for the generated values.
+ Type *Ty;
+ /// If not nullptr, truncate the generated values to TruncToTy.
+ Type *TruncToTy;
const InductionDescriptor &IndDesc;
public:
- VPScalarIVStepsRecipe(PHINode *IV, const InductionDescriptor &IndDesc,
+ VPScalarIVStepsRecipe(Type *Ty, const InductionDescriptor &IndDesc,
VPValue *CanonicalIV, VPValue *Start, VPValue *Step,
- Instruction *Trunc)
+ Type *TruncToTy)
: VPRecipeBase(VPScalarIVStepsSC, {CanonicalIV, Start, Step}),
- VPValue(Trunc ? Trunc : IV, this), IV(IV), IndDesc(IndDesc) {}
+ VPValue(nullptr, this), Ty(Ty), TruncToTy(TruncToTy), IndDesc(IndDesc) {
+ }
~VPScalarIVStepsRecipe() override = default;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 0b02371e62fad..b27016f31cd9d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -368,8 +368,8 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan, Loop &OrigLoop) {
if (R.mayHaveSideEffects() ||
any_of(R.definedValues(),
[](VPValue *V) { return V->getNumUsers() > 0; }) ||
- (R.getUnderlyingInstr() && !isa<VPWidenIntOrFpInductionRecipe>(&R) &&
- !isa<VPScalarIVStepsRecipe>(&R) &&
+ (!isa<VPWidenIntOrFpInductionRecipe>(&R) &&
+ !isa<VPScalarIVStepsRecipe>(&R) && R.getUnderlyingInstr() &&
any_of(R.getUnderlyingInstr()->users(), [&OrigLoop](User *U) {
// Check for live-out users currently not modeled in VPlan.
// Note that exit values of inductions are generated independent of
@@ -405,9 +405,10 @@ void VPlanTransforms::optimizeInductions(VPlan &Plan, ScalarEvolution &SE) {
Step = new VPExpandSCEVRecipe(StepSCEV, SE);
}
+ Instruction *TruncI = IV->getTruncInst();
VPScalarIVStepsRecipe *Steps = new VPScalarIVStepsRecipe(
- IV->getPHINode(), ID, Plan.getCanonicalIV(), IV->getStartValue(), Step,
- IV->getTruncInst());
+ IV->getPHINode()->getType(), ID, Plan.getCanonicalIV(),
+ IV->getStartValue(), Step, TruncI ? TruncI->getType() : nullptr);
HeaderVPBB->insert(Steps, HeaderVPBB->getFirstNonPhi());
if (Step->getDef()) {
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index 847a5da3bcd8b..823d141239930 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -13,11 +13,11 @@ define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x)
; CHECK-NEXT: N1 [label =
; CHECK-NEXT: "for.body:\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION\l" +
-; CHECK-NEXT: " ir\<%iv\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<0\>, ir\<1\>\l" +
-; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, ir\<%iv\>\l" +
+; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<0\>, ir\<1\>\l" +
+; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, vp\<[[STEPS]]\>\l" +
; CHECK-NEXT: " WIDEN ir\<%lv\> = load ir\<%arrayidx\>\l" +
; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>)\l" +
-; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr ir\<%x\>, ir\<%iv\>\l" +
+; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr ir\<%x\>, vp\<[[STEPS]]\>\l" +
; CHECK-NEXT: " WIDEN store ir\<%arrayidx2\>, ir\<%call\>\l" +
; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT:%.+]]\> = VF * UF +(nuw) vp\<[[CAN_IV]]\>\l" +
; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\> vp\<{{.+}}\>\l" +
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 0902dc2c1e229..b2d13ce7652d2 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -14,11 +14,11 @@ define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x)
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[STEPS]]>
; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx>
; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%x>, ir<%iv>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%x>, vp<[[STEPS]]>
; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%call>
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]>
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]>
@@ -99,8 +99,8 @@ define float @print_reduction(i64 %n, float* noalias %y) {
; CHECK-NEXT: for.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
-; CHECK-NEXT: SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[STEPS]]>
; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>)
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]>
@@ -208,13 +208,13 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<4>
-; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr ir<@AB>, ir<0>, ir<%iv>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<4>
+; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr ir<@AB>, ir<0>, vp<[[STEPS]]>
; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0>
; CHECK-NEXT: ir<%AB.0> = load from index 0
; CHECK-NEXT: ir<%AB.1> = load from index 1
; CHECK-NEXT: ir<%AB.3> = load from index 3
-; CHECK-NEXT: CLONE ir<%iv.plus.3> = add ir<%iv>, ir<3>
+; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<[[STEPS]]>, ir<3>
; CHECK-NEXT: WIDEN ir<%add> = add ir<%AB.0>, ir<%AB.1>
; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr ir<@CD>, ir<0>, ir<%iv.plus.3>
; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%gep.CD.3>
@@ -269,10 +269,10 @@ define float @print_fmuladd_strict(float* %a, float* %b, i64 %n) {
; CHECK-NEXT: for.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd>
-; CHECK-NEXT: SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]>
; CHECK-NEXT: WIDEN ir<%l.a> = load ir<%arrayidx>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%b>, ir<%iv>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%b>, vp<[[STEPS]]>
; CHECK-NEXT: WIDEN ir<%l.b> = load ir<%arrayidx2>
; CHECK-NEXT: EMIT vp<[[FMUL:%.]]> = fmul nnan ninf nsz ir<%l.a> ir<%l.b>
; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
@@ -308,8 +308,8 @@ define void @debug_loc_vpinstruction(i32* nocapture %asd, i32* nocapture %bsd) !
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
-; CHECK-NEXT: CLONE ir<%isd> = getelementptr ir<%asd>, ir<%iv>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
+; CHECK-NEXT: CLONE ir<%isd> = getelementptr ir<%asd>, vp<[[STEPS]]>
; CHECK-NEXT: WIDEN ir<%lsd> = load ir<%isd>
; CHECK-NEXT: WIDEN ir<%psd> = add ir<%lsd>, ir<23>
; CHECK-NEXT: WIDEN ir<%cmp1> = icmp ir<%lsd>, ir<100>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index 0583cdf6e71d0..338915ad19c50 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -13,8 +13,8 @@ define void @sink_with_sideeffects(i1 %c, i8* %ptr) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: ir<%tmp0> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
-; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, ir<%tmp0>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1>
+; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]>
; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2>
; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2>
; CHECK-NEXT: CLONE ir<%tmp4> = zext ir<%tmp3>
More information about the llvm-commits
mailing list