[llvm-branch-commits] [llvm] [VPlan] Skip branch term in masks for some preserved uniform edges (PR #218552)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Sep 3 14:08:52 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Andrei Elovikov (eas)
<details>
<summary>Changes</summary>
Only do that for "bypass"-like uniform branch:
```
// Detect patterns like
//
// BB (uniform branch)
// / \
// / +-------------------------+
// | | no other incoming edges |
// | +-------------------------+
// \ / /
// \ / /
// PostDomBB <-- <possible other edges originated before BB
```
I'm not sure if we can generalize it more, but even this limited pattern
should hopefully be enough for an alternative to https://github.com/llvm/llvm-project/pull/141900
implemented on top of partial linearization functionality.
In addition to in-tree tests I also tested this with https://github.com/llvm/llvm-project/pull/220663.
AI-assisted.
---
Patch is 170.71 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218552.diff
34 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+4-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp (+48-3)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+2-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll (+15-34)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll (+34-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll (+1-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll (+4-32)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll (+7-48)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll (+12-18)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll (+2-8)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll (+2-5)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll (+6-10)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll (+25-76)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll (+24-30)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll (+1-17)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll (+7-21)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll (+6-425)
- (modified) llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll (+1-3)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll (+1-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll (+35-8)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll (+19-80)
- (modified) llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll (+2-6)
- (modified) llvm/test/Transforms/LoopVectorize/div-exact.ll (+9-123)
- (modified) llvm/test/Transforms/LoopVectorize/if-pred-stores.ll (+36-129)
- (modified) llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll (-10)
- (modified) llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll (+1-3)
- (modified) llvm/test/Transforms/LoopVectorize/pr37248.ll (-10)
- (modified) llvm/test/Transforms/LoopVectorize/pr45525.ll (+1-4)
- (modified) llvm/test/Transforms/LoopVectorize/predicator.ll (+2-6)
- (modified) llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll (+5-17)
- (modified) llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll (+2-64)
- (modified) llvm/test/Transforms/LoopVectorize/uniform-blend.ll (+1-3)
- (modified) llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll (+2-5)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d85ab84e44bdf..18ed904b40f3f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6251,8 +6251,10 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
case Instruction::UDiv:
case Instruction::SRem:
case Instruction::URem:
- // If not provably safe, use a masked intrinsic.
- if (CM.isPredicatedInst(I))
+ // If not provably safe, use a masked intrinsic. Predicator might have
+ // preserved uniform control flow making this unmasked even if CM expected
+ // it to be masked.
+ if (CM.isPredicatedInst(I) && VPI->isMasked())
return new VPWidenIntrinsicRecipe(
getMaskedDivRemIntrinsic(VPI->getOpcode()), VPI->operands(),
I->getType(), {}, {}, VPI->getDebugLoc());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index da9b2c42984e2..9899b4cd5ee81 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -376,9 +376,54 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// This is the block mask. We OR all unique incoming edges.
for (auto *Predecessor : SetVector<VPBlockBase *>(
VPBB->getPredecessors().begin(), VPBB->getPredecessors().end())) {
- VPValue *EdgeMask = createEdgeMask(cast<VPBasicBlock>(Predecessor), VPBB);
- if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is
- // too.
+ auto *Pred = cast<VPBasicBlock>(Predecessor);
+ bool CanUseBlockMaskOnly = [&] {
+ if (!shouldPreserveTerminator(Pred))
+ return false;
+
+ LLVM_DEBUG(
+ dbgs()
+ << "Checking if can skip branch condition for a uniform branch at "
+ << Pred->getName() << "\n");
+
+ auto False = []([[maybe_unused]] StringRef Reason = "") {
+ LLVM_DEBUG(dbgs() << " can't skip"
+ << (Reason.empty() ? Twine() : (": " + Reason))
+ << "\n");
+ return false;
+ };
+
+ // Detect patterns like
+ //
+ // BB (uniform branch)
+ // / \
+ // / +-------------------------+
+ // | | no other incoming edges |
+ // | +-------------------------+
+ // \ / /
+ // \ / /
+ // PostDomBB <-- <possible other edges originated before BB
+ //
+ // where one of those uniform edges is a "bypass". We can use BB's mask
+ // instead of the edge mask for the masks inside the "box" region above.
+ auto *IPostDomNode = VPPDT.getNode(Pred)->getIDom();
+ if (!IPostDomNode)
+ return False("No post-dom");
+ auto *IPostDom = dyn_cast<VPBasicBlock>(IPostDomNode->getBlock());
+ LLVM_DEBUG(dbgs() << " post-dom: " << IPostDom->getName() << "\n");
+ if (!IPostDom || !VPDT.properlyDominates(Pred, IPostDom))
+ return False("doesn't dominate its post-dom");
+
+ if (!is_contained(Pred->getSuccessors(), IPostDom))
+ return False("not a bypass branch");
+
+ LLVM_DEBUG(dbgs() << " can skip\n");
+ return true;
+ }();
+ VPValue *EdgeMask =
+ CanUseBlockMaskOnly ? getBlockInMask(Pred) : createEdgeMask(Pred, VPBB);
+ if (!EdgeMask) {
+ // Mask of predecessor is all-one so mask of block is too.
setBlockInMask(VPBB, EdgeMask);
return;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b8311f5a77493..f5f8555cb3470 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5433,7 +5433,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
// A predicated access can only be widened (rather than scalarized) if
// the target supports a masked load/store for it.
// TODO: Determine if a load/store needs predication directly in VPlan.
- bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
+ bool IsPredicated =
+ RecipeBuilder.isPredicatedInst(I) && VPI->isMasked();
if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
IsLoad, ScalarTy, getLoadStoreAlignment(I),
getLoadStoreAddressSpace(I)))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
index 78eaa586735d8..d05002391d357 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
@@ -11,53 +11,33 @@ define i32 @test_invariant_replicate_region(i32 %x, i1 %c) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE]]
-; CHECK: [[PRED_UREM_IF]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF2:.*]], label %[[PRED_UREM_CONTINUE1:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE1:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF2:.*]], label %[[PRED_UREM_CONTINUE1]]
; CHECK: [[PRED_UREM_IF2]]:
-; CHECK-NEXT: [[TMP1:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE1]]
; CHECK: [[PRED_UREM_CONTINUE1]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ poison, %[[PRED_UREM_IF]] ], [ [[TMP3]], %[[PRED_UREM_IF2]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF1:.*]], label %[[PRED_UREM_CONTINUE2:.*]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_UREM_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[PRED_UREM_IF1]]:
-; CHECK-NEXT: [[TMP5:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[TMP5]], i64 1
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2:.*]]
; CHECK: [[PRED_UREM_CONTINUE2]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ [[TMP2]], %[[PRED_UREM_CONTINUE1]] ], [ [[TMP4]], %[[PRED_UREM_IF1]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF3:.*]], label %[[PRED_UREM_CONTINUE4:.*]]
-; CHECK: [[PRED_UREM_IF3]]:
-; CHECK-NEXT: [[TMP9:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 2
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4:.*]]
; CHECK: [[PRED_UREM_CONTINUE4]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ [[TMP6]], %[[PRED_UREM_CONTINUE2]] ], [ [[TMP7]], %[[PRED_UREM_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6:.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 96, %[[PRED_UREM_CONTINUE2]] ], [ [[IV_NEXT:%.*]], %[[PRED_UREM_CONTINUE6:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_IF5]]:
; CHECK-NEXT: [[TMP13:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP13]], i64 3
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_CONTINUE6]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_UREM_CONTINUE4]] ], [ [[TMP10]], %[[PRED_UREM_IF5]] ]
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE]]
-; CHECK: [[PRED_UREM_CONTINUE]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_UREM_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[PRED_UREM_CONTINUE6]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP14]], <4 x i32> [[TMP12]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP13]], %[[PRED_UREM_IF5]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 99
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[PRED_UREM_CONTINUE4]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP17:%.*]] = phi i32 [ [[RES]], %[[PRED_UREM_CONTINUE6]] ]
; CHECK-NEXT: ret i32 [[TMP17]]
;
entry:
@@ -84,4 +64,5 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index fdb22beb19695..e7b718dc0d483 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -62,7 +62,34 @@ for.end:
; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
;
-define void @predicated_store(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store(ptr %a, i32 %x, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+ %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
+ %tmp1 = load i32, ptr %tmp0, align 4
+ %tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
+ br i1 %c, label %if.then, label %for.inc
+
+if.then:
+ store i32 %tmp2, ptr %tmp0, align 4
+ br label %for.inc
+
+for.inc:
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+; CHECK-LABEL: store_in_preserved_uniform_control_flow
+; CHECK: Cost of 1 for VF 2: WIDEN store {{.*}}, ir<%tmp2>
+define void @store_in_preserved_uniform_control_flow(ptr %a, i1 %c, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -96,7 +123,7 @@ for.end:
; CHECK: Cost of 0 for VF 2: induction instruction %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %addr, align 4
;
-define void @predicated_store_phi(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_phi(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -105,6 +132,7 @@ for.body:
%addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
%tmp1 = load i32, ptr %addr, align 4
%tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -184,7 +212,7 @@ for.end:
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp2 = add nsw i32 %tmp1, %x
;
-define void @predicated_store_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_scalarized_operand(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -192,6 +220,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -240,7 +269,7 @@ for.end:
; CHECK: Cost of 7 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
;
-define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predication_multi_context(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -248,6 +277,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
index 76aa72c80580b..bc4394d3218ed 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
@@ -81,10 +81,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 %
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = select <vscale x 2 x i1> [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[BROADCAST_SPLAT]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[BROADCAST_SPLAT1]], <vscale x 2 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
index 9efff93543685..3d69698888f16 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
@@ -13,7 +13,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1: [[VECTOR_PH]]:
; VF2IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC1: [[VECTOR_BODY]]:
-; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
@@ -22,20 +22,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_IF]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC1: [[PRED_STORE_IF1]]:
-; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC1: [[PRED_STORE_CONTINUE1]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_IF2]]:
; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_CONTINUE3]]:
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_CONTINUE]]:
; VF2IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC1-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP2]], align 1
; VF2IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -53,7 +44,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
-; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
; VF2IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC2-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP0]], 1
@@ -71,30 +62,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_IF]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC2: [[PRED_STORE_IF1]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC2: [[PRED_STORE_CONTINUE1]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
-; VF2IC2: [[PRED_STORE_IF6]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE7]]
-; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
-; VF2IC2: [[PRED_STORE_IF8]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE9]]
-; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
+; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_IF10]]:
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_CONTINUE11]]:
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_CONTINUE]]:
; VF2IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP6:%.*]] = getelementpt...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218552
More information about the llvm-branch-commits
mailing list