[llvm] [SLP] Fix extract-cost scale using NCD of all external-user sites (PR #199962)
via llvm-commits
llvm-commits at lists.llvm.org
Thu May 28 02:28:05 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
ExtractCostCalculated deduplicates by scalar so only the first
ExternalUser determines the scale, making the cost depend on IR block
ordering via LLVM's reverse-insertion use-list order.
Add a pre-pass computing ScalarToExtractBlock - the nearest common
dominator of all effective extract sites per scalar. For PHI users inside
a loop the effective site is the incoming block; for PHI users outside
all loops it is the PHI's own block (scale = 1). The extract cost is
then scaled by getLoopNestScale of the NCD block, which is fully
order-independent.
Fixes #<!-- -->199548
---
Patch is 26.44 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/199962.diff
3 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+71-2)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/lcssa-phi-extract-scale.ll (+66-44)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/lcssa-phi-inner-loop-scale.ll (+22-16)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9d7821d2760f2..f72207fa148ae 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19551,6 +19551,55 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
}
AllUsersGEPSWithStoresLoads &= UsedLanes.all();
+
+ // Pre-pass: for each externally-used scalar, find the basic block at which
+ // the extractelement will be placed by codegen. This mirrors what
+ // vectorizeTree does: the extract is placed at the nearest common dominator
+ // of all effective use sites. For a non-PHI user the effective site is the
+ // user's own block; for a PHI user it is the incoming block for the scalar
+ // operand (the predecessor of the PHI on the edge that carries the scalar).
+ // Using the NCD of all effective sites rather than the first-encountered
+ // user's block makes the extract-cost scale order-independent and correct
+ // even when users live in different loop nests.
+ SmallDenseMap<Value *, BasicBlock *> ScalarToExtractBlock;
+ for (const ExternalUser &EU : ExternalUses) {
+ if (!EU.User || isa<InsertElementInst>(EU.User))
+ continue;
+ if (EphValues.count(EU.User))
+ continue;
+ BasicBlock *UserParent = cast<Instruction>(EU.User)->getParent();
+ if (!DT->isReachableFromEntry(UserParent) || UserParent->isEHPad() ||
+ isa_and_present<UnreachableInst>(UserParent->getTerminator()))
+ continue;
+ BasicBlock *UseBlock = nullptr;
+ if (auto *PHI = dyn_cast<PHINode>(EU.User)) {
+ // When the PHI itself is inside a loop, the extractelement is placed
+ // in the incoming block for the scalar operand (the predecessor edge),
+ // not in the PHI's own block. This applies to LCSSA phis at an inner-
+ // loop exit that are still inside an outer loop: the incoming block is
+ // in the inner loop while the PHI block is in the outer loop.
+ // When the PHI is outside all loops (a true loop-exit phi), codegen
+ // uses a vector phi at the exit block and the extract stays there
+ // (scale = 1), so we keep the PHI's own block as the effective site.
+ if (LI->getLoopFor(PHI->getParent())) {
+ for (unsigned Idx : seq<unsigned>(PHI->getNumIncomingValues())) {
+ if (PHI->getIncomingValue(Idx) != EU.Scalar)
+ continue;
+ BasicBlock *InBB = PHI->getIncomingBlock(Idx);
+ UseBlock =
+ UseBlock ? DT->findNearestCommonDominator(UseBlock, InBB) : InBB;
+ }
+ }
+ if (!UseBlock)
+ UseBlock = cast<Instruction>(EU.User)->getParent();
+ } else {
+ UseBlock = cast<Instruction>(EU.User)->getParent();
+ }
+ auto [It, Inserted] = ScalarToExtractBlock.try_emplace(EU.Scalar, UseBlock);
+ if (!Inserted && It->second && UseBlock)
+ It->second = DT->findNearestCommonDominator(It->second, UseBlock);
+ }
+
SmallDenseSet<std::pair<Value *, Value *>, 8> CheckedScalarUser;
for (ExternalUser &EU : ExternalUses) {
LLVM_DEBUG(dbgs() << "SLP: Computing cost for external use of TreeEntry "
@@ -19836,8 +19885,28 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
}
- ExtraCost = ScaleCost(ExtraCost, *Entry, EU.Scalar,
- cast_or_null<Instruction>(EU.User));
+ // Scale the extract cost by the execution frequency of the block where
+ // codegen will place the extractelement. That block is the nearest common
+ // dominator of all effective use sites (precomputed in ScalarToExtractBlock
+ // above), which is order-independent. For scalars kept as originals the
+ // existing ScaleCost path (user-block based) remains correct, since the
+ // scalar instruction executes at its definition site's frequency.
+ if (!ExternalUsesAsOriginalScalar.contains(EU.Scalar)) {
+ if (ExtraCost.isValid() && ExtraCost != 0) {
+ BasicBlock *ExtractBB = ScalarToExtractBlock.lookup(EU.Scalar);
+ if (const Loop *L = ExtractBB ? LI->getLoopFor(ExtractBB) : nullptr) {
+ uint64_t Scale = getLoopNestScale(
+ findInnermostNonInvariantLoop(L, ArrayRef<Value *>(EU.Scalar)));
+ LLVM_DEBUG(dbgs()
+ << "SLP: Extract scale " << Scale << " (NCD block) for "
+ << EU.Scalar->getNameOrAsOperand() << "\n");
+ ExtraCost *= Scale;
+ }
+ }
+ } else {
+ ExtraCost = ScaleCost(ExtraCost, *Entry, EU.Scalar,
+ cast_or_null<Instruction>(EU.User));
+ }
ExtractCost += ExtraCost;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/lcssa-phi-extract-scale.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/lcssa-phi-extract-scale.ll
index 4ad0e6934d7a5..9b8520c122066 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/lcssa-phi-extract-scale.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/lcssa-phi-extract-scale.ll
@@ -9,35 +9,43 @@ define double @test(ptr %obj, ptr %arr, i32 %n) {
; CHECK-SAME: ptr [[OBJ:%.*]], ptr [[ARR:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[POS1:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 44
+; CHECK-NEXT: [[POS2:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 48
; CHECK-NEXT: [[COND_ENTER:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-NEXT: br i1 [[COND_ENTER]], label %[[OUTER_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK: [[OUTER_PREHEADER]]:
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[LOCAL_4_128:%.*]] = phi i32 [ 0, %[[OUTER_PREHEADER]] ], [ [[LCSSA_IV:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: %"$vphi.1" = phi i32 [ 16, %[[OUTER_PREHEADER]] ], [ [[DOTSINK6_I_LCSSA:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: %"$vphi.0" = phi i32 [ 4, %[[OUTER_PREHEADER]] ], [ [[DOTSINK_I_LCSSA:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[LOCAL_4_128:%.*]] = phi i32 [ 0, %[[OUTER_PREHEADER]] ], [ [[LCSSA_IV:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT: [[LOCAL_3_127:%.*]] = phi i32 [ 0, %[[OUTER_PREHEADER]] ], [ [[LOCAL_3_16_LCSSA:%.*]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i32> [ <i32 4, i32 16>, %[[OUTER_PREHEADER]] ], [ [[TMP15:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT: br label %[[INNER_HEADER:.*]]
; CHECK: [[INNER_HEADER]]:
-; CHECK-NEXT: [[LOCAL_4_128420:%.*]] = phi i32 [ [[LOCAL_4_128]], %[[OUTER_HEADER]] ], [ [[INC:%.*]], %[[INNER_BODY:.*]] ]
+; CHECK-NEXT: %"$vphi.1418" = phi i32 [ %"$vphi.1", %[[OUTER_HEADER]] ], [ [[DOTSINK6_I:%.*]], %[[INNER_BODY:.*]] ]
+; CHECK-NEXT: %"$vphi.0419" = phi i32 [ %"$vphi.0", %[[OUTER_HEADER]] ], [ [[DOTSINK_I:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT: [[LOCAL_4_128420:%.*]] = phi i32 [ [[LOCAL_4_128]], %[[OUTER_HEADER]] ], [ [[INC:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[LOCAL_3_127421:%.*]] = phi i32 [ [[LOCAL_3_127]], %[[OUTER_HEADER]] ], [ [[LOCAL_3_16:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[CHUNK_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[CHUNK_IV_NEXT:%.*]], %[[INNER_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ [[TMP0]], %[[OUTER_HEADER]] ], [ [[TMP14:%.*]], %[[INNER_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], splat (i32 -1)
-; CHECK-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> splat (i32 16), <2 x i32> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP5]], <4 x i32> <i32 1, i32 0, i32 2, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult <4 x i32> [[TMP6]], splat (i32 17)
-; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[C27:%.*]] = icmp ult i32 %"$vphi.0419", 17
+; CHECK-NEXT: [[C28:%.*]] = icmp ult i32 %"$vphi.1418", 17
+; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[C28]], [[C27]]
+; CHECK-NEXT: [[DOTNOT3_I17:%.*]] = icmp eq i32 %"$vphi.0419", 0
+; CHECK-NEXT: [[V29:%.*]] = add i32 %"$vphi.0419", -1
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[DOTNOT3_I17]], i32 16, i32 [[V29]]
+; CHECK-NEXT: [[C30:%.*]] = icmp ult i32 [[TMP10]], 17
+; CHECK-NEXT: [[WIDE_CHK354:%.*]] = and i1 [[WIDE_CHK]], [[C30]]
+; CHECK-NEXT: [[DOTNOT4_I19:%.*]] = icmp eq i32 %"$vphi.1418", 0
+; CHECK-NEXT: [[V31:%.*]] = add i32 %"$vphi.1418", -1
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[DOTNOT4_I19]], i32 16, i32 [[V31]]
+; CHECK-NEXT: [[C32:%.*]] = icmp ult i32 [[TMP11]], 17
+; CHECK-NEXT: [[TMP8:%.*]] = and i1 [[C32]], [[WIDE_CHK354]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[INNER_BODY]], label %[[DEOPT:.*]]
; CHECK: [[INNER_BODY]]:
; CHECK-NEXT: [[CHUNK_IV_NEXT]] = add nuw nsw i64 [[CHUNK_IV]], 1
-; CHECK-NEXT: [[V34:%.*]] = zext nneg i32 [[TMP2]] to i64
+; CHECK-NEXT: [[V34:%.*]] = zext nneg i32 %"$vphi.0419" to i64
; CHECK-NEXT: [[V35:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V34]]
; CHECK-NEXT: [[V36:%.*]] = load i32, ptr [[V35]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[V37:%.*]] = zext nneg i32 [[TMP9]] to i64
+; CHECK-NEXT: [[V37:%.*]] = zext nneg i32 %"$vphi.1418" to i64
; CHECK-NEXT: [[V38:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V37]]
; CHECK-NEXT: [[V39:%.*]] = load i32, ptr [[V38]], align 4
; CHECK-NEXT: [[V40:%.*]] = sub i32 [[V36]], [[V39]]
@@ -47,11 +55,9 @@ define double @test(ptr %obj, ptr %arr, i32 %n) {
; CHECK-NEXT: store i32 [[SPEC_SELECT_I13]], ptr [[V38]], align 4
; CHECK-NEXT: [[V43:%.*]] = sitofp i32 [[SPEC_SELECT_I13]] to double
; CHECK-NEXT: [[V44:%.*]] = fmul nnan double [[V43]], f0x3E00000000200000
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP5]], i32 0
; CHECK-NEXT: [[V45:%.*]] = zext nneg i32 [[TMP10]] to i64
; CHECK-NEXT: [[V46:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V45]]
; CHECK-NEXT: [[V47:%.*]] = load i32, ptr [[V46]], align 4
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP5]], i32 1
; CHECK-NEXT: [[V48:%.*]] = zext nneg i32 [[TMP11]] to i64
; CHECK-NEXT: [[V49:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V48]]
; CHECK-NEXT: [[V50:%.*]] = load i32, ptr [[V49]], align 4
@@ -60,10 +66,14 @@ define double @test(ptr %obj, ptr %arr, i32 %n) {
; CHECK-NEXT: [[V53:%.*]] = icmp slt i32 [[V51]], 0
; CHECK-NEXT: [[SPEC_SELECT_I:%.*]] = select i1 [[V53]], i32 [[V52]], i32 [[V51]]
; CHECK-NEXT: store i32 [[SPEC_SELECT_I]], ptr [[V49]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <2 x i32> [[TMP5]], zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = add nsw <2 x i32> [[TMP5]], splat (i32 -1)
-; CHECK-NEXT: [[TMP14]] = select <2 x i1> [[TMP12]], <2 x i32> splat (i32 16), <2 x i32> [[TMP13]]
-; CHECK-NEXT: store <2 x i32> [[TMP14]], ptr [[POS1]], align 4
+; CHECK-NEXT: [[DOTNOT3_I:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT: [[V54:%.*]] = add nsw i32 [[TMP10]], -1
+; CHECK-NEXT: [[DOTSINK_I]] = select i1 [[DOTNOT3_I]], i32 16, i32 [[V54]]
+; CHECK-NEXT: store i32 [[DOTSINK_I]], ptr [[POS1]], align 4
+; CHECK-NEXT: [[DOTNOT4_I:%.*]] = icmp eq i32 [[TMP11]], 0
+; CHECK-NEXT: [[V55:%.*]] = add nsw i32 [[TMP11]], -1
+; CHECK-NEXT: [[DOTSINK6_I]] = select i1 [[DOTNOT4_I]], i32 16, i32 [[V55]]
+; CHECK-NEXT: store i32 [[DOTSINK6_I]], ptr [[POS2]], align 8
; CHECK-NEXT: [[V56:%.*]] = sitofp i32 [[SPEC_SELECT_I]] to double
; CHECK-NEXT: [[V57:%.*]] = fmul nnan double [[V56]], f0x3E00000000200000
; CHECK-NEXT: [[V58:%.*]] = fmul nnan double [[V44]], [[V44]]
@@ -78,16 +88,17 @@ define double @test(ptr %obj, ptr %arr, i32 %n) {
; CHECK-NEXT: [[V64:%.*]] = and i1 [[DOTNOT]], [[CHUNK_COND]]
; CHECK-NEXT: br i1 [[V64]], label %[[INNER_HEADER]], label %[[OUTER_LATCH]]
; CHECK: [[DEOPT]]:
-; CHECK-NEXT: %"$vphi.0419.lcssa" = phi i32 [ [[TMP2]], %[[INNER_HEADER]] ]
+; CHECK-NEXT: %"$vphi.0419.lcssa" = phi i32 [ %"$vphi.0419", %[[INNER_HEADER]] ]
; CHECK-NEXT: [[LOCAL_4_128420_LCSSA:%.*]] = phi i32 [ [[LOCAL_4_128420]], %[[INNER_HEADER]] ]
; CHECK-NEXT: [[LOCAL_3_127421_LCSSA:%.*]] = phi i32 [ [[LOCAL_3_127421]], %[[INNER_HEADER]] ]
; CHECK-NEXT: call void @sink_use(i32 %"$vphi.0419.lcssa", i32 [[LOCAL_4_128420_LCSSA]], i32 [[LOCAL_3_127421_LCSSA]])
; CHECK-NEXT: ret double 0.000000e+00
; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[DOTSINK_I_LCSSA]] = phi i32 [ [[DOTSINK_I]], %[[INNER_BODY]] ]
+; CHECK-NEXT: [[DOTSINK6_I_LCSSA]] = phi i32 [ [[DOTSINK6_I]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[LOCAL_3_16_LCSSA]] = phi i32 [ [[LOCAL_3_16]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[LCSSA_IV]] = phi i32 [ [[INC]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[DOTNOT_LCSSA:%.*]] = phi i1 [ [[DOTNOT]], %[[INNER_BODY]] ]
-; CHECK-NEXT: [[TMP15]] = phi <2 x i32> [ [[TMP14]], %[[INNER_BODY]] ]
; CHECK-NEXT: br i1 [[DOTNOT_LCSSA]], label %[[OUTER_HEADER]], label %[[EXIT_LOOP:.*]]
; CHECK: [[EXIT_LOOP]]:
; CHECK-NEXT: [[FINAL:%.*]] = phi i32 [ [[LOCAL_3_16_LCSSA]], %[[OUTER_LATCH]] ]
@@ -218,33 +229,43 @@ define double @test_inloop_first(ptr %obj, ptr %arr, i32 %n) {
; CHECK-SAME: ptr [[OBJ:%.*]], ptr [[ARR:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[POS1:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 44
+; CHECK-NEXT: [[POS2:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 48
; CHECK-NEXT: [[COND_ENTER:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-NEXT: br i1 [[COND_ENTER]], label %[[OUTER_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK: [[OUTER_PREHEADER]]:
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[LOCAL_4_128:%.*]] = phi i32 [ 0, %[[OUTER_PREHEADER]] ], [ [[LCSSA_IV:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: %"$vphi.1" = phi i32 [ 16, %[[OUTER_PREHEADER]] ], [ [[DOTSINK6_I_LCSSA:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: %"$vphi.0" = phi i32 [ 4, %[[OUTER_PREHEADER]] ], [ [[DOTSINK_I_LCSSA:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[LOCAL_4_128:%.*]] = phi i32 [ 0, %[[OUTER_PREHEADER]] ], [ [[LCSSA_IV:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT: [[LOCAL_3_127:%.*]] = phi i32 [ 0, %[[OUTER_PREHEADER]] ], [ [[LOCAL_3_16_LCSSA:%.*]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i32> [ <i32 4, i32 16>, %[[OUTER_PREHEADER]] ], [ [[TMP9:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT: br label %[[INNER_HEADER:.*]]
; CHECK: [[INNER_HEADER]]:
-; CHECK-NEXT: [[LOCAL_4_128420:%.*]] = phi i32 [ [[LOCAL_4_128]], %[[OUTER_HEADER]] ], [ [[INC:%.*]], %[[INNER_BODY:.*]] ]
+; CHECK-NEXT: %"$vphi.1418" = phi i32 [ %"$vphi.1", %[[OUTER_HEADER]] ], [ [[DOTSINK6_I:%.*]], %[[INNER_BODY:.*]] ]
+; CHECK-NEXT: %"$vphi.0419" = phi i32 [ %"$vphi.0", %[[OUTER_HEADER]] ], [ [[DOTSINK_I:%.*]], %[[INNER_BODY]] ]
+; CHECK-NEXT: [[LOCAL_4_128420:%.*]] = phi i32 [ [[LOCAL_4_128]], %[[OUTER_HEADER]] ], [ [[INC:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[LOCAL_3_127421:%.*]] = phi i32 [ [[LOCAL_3_127]], %[[OUTER_HEADER]] ], [ [[LOCAL_3_16:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[CHUNK_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[CHUNK_IV_NEXT:%.*]], %[[INNER_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ [[TMP0]], %[[OUTER_HEADER]] ], [ [[TMP15:%.*]], %[[INNER_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], splat (i32 -1)
-; CHECK-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> splat (i32 16), <2 x i32> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP5]], <4 x i32> <i32 1, i32 0, i32 2, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult <4 x i32> [[TMP6]], splat (i32 17)
-; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[C27:%.*]] = icmp ult i32 %"$vphi.0419", 17
+; CHECK-NEXT: [[C28:%.*]] = icmp ult i32 %"$vphi.1418", 17
+; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[C28]], [[C27]]
+; CHECK-NEXT: [[DOTNOT3_I17:%.*]] = icmp eq i32 %"$vphi.0419", 0
+; CHECK-NEXT: [[V29:%.*]] = add i32 %"$vphi.0419", -1
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[DOTNOT3_I17]], i32 16, i32 [[V29]]
+; CHECK-NEXT: [[C30:%.*]] = icmp ult i32 [[TMP11]], 17
+; CHECK-NEXT: [[WIDE_CHK354:%.*]] = and i1 [[WIDE_CHK]], [[C30]]
+; CHECK-NEXT: [[DOTNOT4_I19:%.*]] = icmp eq i32 %"$vphi.1418", 0
+; CHECK-NEXT: [[V31:%.*]] = add i32 %"$vphi.1418", -1
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[DOTNOT4_I19]], i32 16, i32 [[V31]]
+; CHECK-NEXT: [[C32:%.*]] = icmp ult i32 [[TMP12]], 17
+; CHECK-NEXT: [[TMP8:%.*]] = and i1 [[C32]], [[WIDE_CHK354]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[INNER_BODY]], label %[[DEOPT:.*]]
; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[DOTSINK_I_LCSSA]] = phi i32 [ [[DOTSINK_I]], %[[INNER_BODY]] ]
+; CHECK-NEXT: [[DOTSINK6_I_LCSSA]] = phi i32 [ [[DOTSINK6_I]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[LOCAL_3_16_LCSSA]] = phi i32 [ [[LOCAL_3_16]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[LCSSA_IV]] = phi i32 [ [[INC]], %[[INNER_BODY]] ]
; CHECK-NEXT: [[DOTNOT_LCSSA:%.*]] = phi i1 [ [[DOTNOT:%.*]], %[[INNER_BODY]] ]
-; CHECK-NEXT: [[TMP9]] = phi <2 x i32> [ [[TMP15]], %[[INNER_BODY]] ]
; CHECK-NEXT: br i1 [[DOTNOT_LCSSA]], label %[[OUTER_HEADER]], label %[[EXIT_LOOP:.*]]
; CHECK: [[EXIT_LOOP]]:
; CHECK-NEXT: [[FINAL:%.*]] = phi i32 [ [[LOCAL_3_16_LCSSA]], %[[OUTER_LATCH]] ]
@@ -252,11 +273,10 @@ define double @test_inloop_first(ptr %obj, ptr %arr, i32 %n) {
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[INNER_BODY]]:
; CHECK-NEXT: [[CHUNK_IV_NEXT]] = add nuw nsw i64 [[CHUNK_IV]], 1
-; CHECK-NEXT: [[V34:%.*]] = zext nneg i32 [[TMP2]] to i64
+; CHECK-NEXT: [[V34:%.*]] = zext nneg i32 %"$vphi.0419" to i64
; CHECK-NEXT: [[V35:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V34]]
; CHECK-NEXT: [[V36:%.*]] = load i32, ptr [[V35]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[V37:%.*]] = zext nneg i32 [[TMP10]] to i64
+; CHECK-NEXT: [[V37:%.*]] = zext nneg i32 %"$vphi.1418" to i64
; CHECK-NEXT: [[V38:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V37]]
; CHECK-NEXT: [[V39:%.*]] = load i32, ptr [[V38]], align 4
; CHECK-NEXT: [[V40:%.*]] = sub i32 [[V36]], [[V39]]
@@ -266,11 +286,9 @@ define double @test_inloop_first(ptr %obj, ptr %arr, i32 %n) {
; CHECK-NEXT: store i32 [[SPEC_SELECT_I13]], ptr [[V38]], align 4
; CHECK-NEXT: [[V43:%.*]] = sitofp i32 [[SPEC_SELECT_I13]] to double
; CHECK-NEXT: [[V44:%.*]] = fmul nnan double [[V43]], f0x3E00000000200000
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP5]], i32 0
; CHECK-NEXT: [[V45:%.*]] = zext nneg i32 [[TMP11]] to i64
; CHECK-NEXT: [[V46:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V45]]
; CHECK-NEXT: [[V47:%.*]] = load i32, ptr [[V46]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[TMP5]], i32 1
; CHECK-NEXT: [[V48:%.*]] = zext nneg i32 [[TMP12]] to i64
; CHECK-NEXT: [[V49:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARR]], i64 [[V48]]
; CHECK-NEXT: [[V50:%.*]] = load i32, ptr [[V49]], align 4
@@ -279,10 +297,14 @@ define double @test_inloop_first(ptr %obj, ptr %arr, i32 %n) {
; CHECK-NEXT: [[V53:%.*]] = icmp slt i32 [[V51]], 0
; CHECK-NEXT: [[SPEC_SELECT_I:%.*]] = select i1 [[V53]], i32 [[V52]], i32 [[V51]]
; CHECK-NEXT: store i32 [[SPEC_SELECT_I]], ptr [[V49]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <2 x i32> [[TMP5]], zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = add nsw <2 x i32> [[TMP5]], splat (i32 -1)
-; CHECK-NEXT: [[TMP15]] = select <2 x i1> [[TMP13]], <2 x i32> splat (i32 16), <2 x i32> [[TMP14]]
-; CHECK-NEXT: store <2 x i32> [[TMP15]], ptr [[POS1]], align 4
+; CHECK-NEXT: [[DOTNOT3_I:%.*]] = icmp eq i32 [[TMP11]], 0
+; CHECK-NEXT: [[V54:%.*]] = add nsw i32 [[TMP1...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/199962
More information about the llvm-commits
mailing list