[llvm] cb4efe1 - [VPlan] Don't trigger VF assertion if VPlan has extra simplifications.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 22 13:38:25 PDT 2024
Author: Florian Hahn
Date: 2024-08-22T21:38:06+01:00
New Revision: cb4efe1d078144a72306732a56afea3885650e8d
URL: https://github.com/llvm/llvm-project/commit/cb4efe1d078144a72306732a56afea3885650e8d
DIFF: https://github.com/llvm/llvm-project/commit/cb4efe1d078144a72306732a56afea3885650e8d.diff
LOG: [VPlan] Don't trigger VF assertion if VPlan has extra simplifications.
There are cases where VPlans contain some simplifications that are very
hard to accurately account for up-front in the legacy cost model. Those
cases are caused by un-simplified inputs, which trigger the assert
ensuring both the legacy and VPlan-based cost model agree on the VF.
To avoid false positives due to missed simplifications in general, only
trigger the assert if the chosen VPlan doesn't contain any additional
simplifications.
Fixes https://github.com/llvm/llvm-project/issues/104714.
Fixes https://github.com/llvm/llvm-project/issues/105713.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8e9324ba718b39..32e8f331257801 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7237,6 +7237,56 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
return Cost;
}
+/// Return true if the original loop \ TheLoop contains any instructions that do
+/// not have corresponding recipes in \p Plan and are not marked to be ignored
+/// in \p CostCtx. This means the VPlan contains simplification that the legacy
+/// cost-model did not account for.
+static bool
+planContainsAdditionalSimplifications(VPlan &Plan, ElementCount VF,
+ VPCostContext &CostCtx, Loop *TheLoop,
+ LoopVectorizationCostModel &CM) {
+ // First collect all instructions for the recipes in Plan.
+ auto GetInstructionForCost = [](const VPRecipeBase *R) -> Instruction q {
+ if (auto *S = dyn_cast<VPSingleDefRecipe>(R))
+ return dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
+ if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
+ return &WidenMem->getIngredient();
+ return nullptr;
+ };
+
+ DenseSet<Instruction *> SeenInstrs;
+ auto Iter = vp_depth_first_deep(Plan.getEntry());
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
+ for (VPRecipeBase &R : *VPBB) {
+ if (auto *IR = dyn_cast<VPInterleaveRecipe>(&R)) {
+ auto *IG = IR->getInterleaveGroup();
+ unsigned NumMembers = IG->getNumMembers();
+ for (unsigned I = 0; I != NumMembers; ++I) {
+ if (Instruction *M = IG->getMember(I))
+ SeenInstrs.insert(M);
+ }
+ continue;
+ }
+ if (Instruction *UI = GetInstructionForCost(&R))
+ SeenInstrs.insert(UI);
+ }
+ }
+
+ // Return true if the loop contains any instructions that are not also part of
+ // the VPlan or are skipped for VPlan-based cost computations. This indicates
+ // that the VPlan contains extra simplifications.
+ return any_of(
+ TheLoop->blocks(), [&SeenInstrs, VF, &CostCtx, &CM](BasicBlock *BB) {
+ return any_of(*BB, [&SeenInstrs, VF, &CostCtx, &CM](Instruction &I) {
+ if (isa<PHINode>(&I))
+ return false;
+ return !SeenInstrs.contains(&I) &&
+ !CostCtx.skipCostComputation(&I, true) &&
+ !CM.canTruncateToMinimalBitwidth(&I, VF);
+ });
+ });
+}
+
VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
return VectorizationFactor::Disabled();
@@ -7292,7 +7342,20 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
// cost-model and will be retired once the VPlan-based cost-model is
// stabilized.
VectorizationFactor LegacyVF = selectVectorizationFactor();
- assert(BestFactor.Width == LegacyVF.Width &&
+ VPlan &BestPlan = getPlanFor(BestFactor.Width);
+
+ // Pre-compute the cost and use it to check if BestPlan contains any
+ // simplifications not accounted for in the legacy cost model. If that's the
+ // case, don't trigger the assertion, as the extra simplifications may cause a
+ //
diff erent VF to be picked by the VPlan-based cost model.
+ LLVMContext &LLVMCtx = OrigLoop->getHeader()->getContext();
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
+ LLVMCtx, CM);
+ precomputeCosts(BestPlan, BestFactor.Width, CostCtx);
+ assert((BestFactor.Width == LegacyVF.Width ||
+ planContainsAdditionalSimplifications(getPlanFor(BestFactor.Width),
+ BestFactor.Width, CostCtx,
+ OrigLoop, CM)) &&
" VPlan cost model and legacy cost model disagreed");
assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
"when vectorizing, the scalar cost must be computed.");
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
index d970b427d035da..b629b964a70c04 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -827,6 +827,120 @@ exit:
ret void
}
+; Test case for https://github.com/llvm/llvm-project/issues/100591.
+define void @dead_load_in_block(ptr %dst, ptr %src, i8 %N, i64 %x) #0 {
+; CHECK-LABEL: define void @dead_load_in_block(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i8 [[N:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[N_EXT:%.*]] = zext i8 [[N]] to i64
+; CHECK-NEXT: [[UMIN7:%.*]] = call i64 @llvm.umin.i64(i64 [[N_EXT]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[N_EXT]], [[UMIN7]]
+; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[UMIN7]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umax.i64(i64 40, i64 [[TMP5]])
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP6]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N_EXT]], i64 1)
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N_EXT]], [[UMIN]]
+; CHECK-NEXT: [[TMP8:%.*]] = udiv i64 [[TMP7]], 3
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[UMIN]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 12
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = shl i64 [[X]], 2
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 4
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP15]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2
+; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-NEXT: [[TMP19:%.*]] = add <vscale x 2 x i64> [[TMP18]], zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP20]]
+; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 2
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 3, [[TMP22]]
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP23]], i64 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 2 x i64> [[VEC_IND]]
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> [[TMP24]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)), !alias.scope [[META18:![0-9]+]], !noalias [[META21:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[L_0:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEXT: [[C_0:%.*]] = icmp eq i32 [[L_0]], 0
+; CHECK-NEXT: br i1 [[C_0]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[GEP_SRC_X:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[X]]
+; CHECK-NEXT: [[L_DEAD:%.*]] = load i32, ptr [[GEP_SRC_X]], align 4
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 3
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %N.ext = zext i8 %N to i64
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %l.0 = load i32, ptr %src, align 4
+ %c.0 = icmp eq i32 %l.0, 0
+ br i1 %c.0, label %loop.latch , label %then
+
+then:
+ %gep.src.x = getelementptr i32, ptr %src, i64 %x
+ %l.dead = load i32, ptr %gep.src.x, align 4
+ br label %loop.latch
+
+loop.latch:
+ %gep.dst = getelementptr i32, ptr %dst, i64 %iv
+ store i32 0, ptr %gep.dst, align 4
+ %iv.next = add i64 %iv, 3
+ %cmp = icmp ult i64 %iv, %N.ext
+ br i1 %cmp, label %loop.header, label %exit
+
+exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+64bit,+v" }
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
@@ -847,4 +961,12 @@ attributes #0 = { "target-features"="+64bit,+v" }
; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
+; CHECK: [[META18]] = !{[[META19:![0-9]+]]}
+; CHECK: [[META19]] = distinct !{[[META19]], [[META20:![0-9]+]]}
+; CHECK: [[META20]] = distinct !{[[META20]], !"LVerDomain"}
+; CHECK: [[META21]] = !{[[META22:![0-9]+]], [[META23:![0-9]+]]}
+; CHECK: [[META22]] = distinct !{[[META22]], [[META20]]}
+; CHECK: [[META23]] = distinct !{[[META23]], [[META20]]}
+; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META1]]}
;.
More information about the llvm-commits
mailing list