[llvm-branch-commits] [llvm] [VPlan] Reassociate header mask in the predicator during mask creation (PR #219061)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 28 16:52:13 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/219061
>From 3a3724bbf03add227d455b22dff83eebff020ce6 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 08:53:10 -0700
Subject: [PATCH 1/3] Copy tests to predicator.ll
---
.../LoopVectorize/VPlan/predicator.ll | 343 ++++++++++++++++++
1 file changed, 343 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 483ab8ad94b50..16f340818345f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -959,3 +959,346 @@ latch:
exit:
ret void
}
+
+; RUN: llvm-extract --func const_tc_with_predicated_store < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vplan-verify-each -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix TMP1
+define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 {
+; CHECK-LABEL: VPlan for loop in 'const_tc_with_predicated_store'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): if.then
+; CHECK-EMPTY:
+; CHECK-NEXT: if.then:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: Successor(s): if.else1
+; CHECK-EMPTY:
+; CHECK-NEXT: if.else1:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP6]]>, ir<%c1>
+; CHECK-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP6]]> ir<1.000000e+00>/ir<%c1>
+; CHECK-NEXT: Successor(s): if.else2
+; CHECK-EMPTY:
+; CHECK-NEXT: if.else2:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c3>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c3>
+; CHECK-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP11]]> ir<%phi1>/vp<[[VP12]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; TMP1-LABEL: VPlan for loop in 'const_tc_with_predicated_store'
+; TMP1: VPlan ' for UF>=1' {
+; TMP1-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; TMP1-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; TMP1-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; TMP1-NEXT: Live-in ir<57> = original trip-count
+; TMP1-EMPTY:
+; TMP1-NEXT: ir-bb<entry>:
+; TMP1-NEXT: Successor(s): scalar.ph, vector.ph
+; TMP1-EMPTY:
+; TMP1-NEXT: vector.ph:
+; TMP1-NEXT: Successor(s): vector loop
+; TMP1-EMPTY:
+; TMP1-NEXT: <x1> vector loop: {
+; TMP1-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; TMP1-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
+; TMP1-EMPTY:
+; TMP1-NEXT: vector.body:
+; TMP1-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; TMP1-NEXT: Successor(s): vector.body.split
+; TMP1-EMPTY:
+; TMP1-NEXT: vector.body.split:
+; TMP1-NEXT: Successor(s): if.then
+; TMP1-EMPTY:
+; TMP1-NEXT: if.then:
+; TMP1-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; TMP1-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; TMP1-NEXT: Successor(s): if.else1
+; TMP1-EMPTY:
+; TMP1-NEXT: if.else1:
+; TMP1-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2>
+; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP7]]>
+; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP9]]>
+; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP8]]> ir<1.000000e+00>/vp<[[VP9]]>
+; TMP1-NEXT: Successor(s): if.else2
+; TMP1-EMPTY:
+; TMP1-NEXT: if.else2:
+; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = not ir<%c3>
+; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP10]]>, vp<[[VP11]]>
+; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%c2>
+; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP13]]>
+; TMP1-NEXT: Successor(s): latch
+; TMP1-EMPTY:
+; TMP1-NEXT: latch:
+; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP10]]>, ir<%c3>
+; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP14]]> ir<%phi1>/vp<[[VP15]]>
+; TMP1-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv>
+; TMP1-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP4]]>
+; TMP1-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; TMP1-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56>, vp<[[VP4]]>
+; TMP1-NEXT: Successor(s): vector.latch
+; TMP1-EMPTY:
+; TMP1-NEXT: vector.latch:
+; TMP1-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; TMP1-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; TMP1-NEXT: No successors
+; TMP1-NEXT: }
+; TMP1-NEXT: Successor(s): middle.block
+; TMP1-EMPTY:
+; TMP1-NEXT: middle.block:
+; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; TMP1-NEXT: EMIT vp<%cmp.n> = icmp eq ir<57>, vp<[[VP2]]>
+; TMP1-NEXT: EMIT branch-on-cond ir<true>
+; TMP1-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; TMP1-EMPTY:
+; TMP1-NEXT: ir-bb<exit>:
+; TMP1-NEXT: No successors
+; TMP1-EMPTY:
+; TMP1-NEXT: scalar.ph:
+; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP17]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; TMP1-NEXT: Successor(s): ir-bb<header>
+; TMP1-EMPTY:
+; TMP1-NEXT: ir-bb<header>:
+; TMP1-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; TMP1-NEXT: No successors
+; TMP1-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ br i1 %c1, label %if.else1, label %if.then
+
+if.then:
+ br i1 %c2, label %if.else2, label %if.else1
+
+if.else1:
+ %phi1 = phi float [ 0.0, %if.then ], [ 1.0, %header ]
+ br i1 %c3, label %latch, label %if.else2
+
+if.else2:
+ br label %latch
+
+latch:
+ %phi = phi float [ %phi1, %if.else1 ], [ 2.0, %if.else2 ]
+ %gep = getelementptr float, ptr %dst, i64 %iv
+ store float %phi, ptr %gep, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv, 56
+ br i1 %ec, label %exit, label %header
+
+exit:
+ ret void
+}
+
+; RUN: llvm-extract --func test < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix=TMP2
+define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src2, ptr noalias %dst, i1 %c1, i1 %c2, i1 %c3) {
+; CHECK-LABEL: VPlan for loop in 'test'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): check.cond1
+; CHECK-EMPTY:
+; CHECK-NEXT: check.cond1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): load.v0
+; CHECK-EMPTY:
+; CHECK-NEXT: load.v0:
+; CHECK-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, ir<%c1>
+; CHECK-NEXT: Successor(s): load.v1
+; CHECK-EMPTY:
+; CHECK-NEXT: load.v1:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%cond1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or ir<%c1>, vp<[[VP5]]>
+; CHECK-NEXT: BLEND ir<%val0> = ir<%v0>/ir<%c1> ir<0>/vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): load.v2.check
+; CHECK-EMPTY:
+; CHECK-NEXT: load.v2.check:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cond1>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP6]]> ir<0>/vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): load.v2
+; CHECK-EMPTY:
+; CHECK-NEXT: load.v2:
+; CHECK-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, ir<%c3>
+; CHECK-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, ir<%c3>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%c3>
+; CHECK-NEXT: BLEND ir<%result> = ir<%val3>/ir<%c3> ir<%val2>/vp<[[VP9]]>
+; CHECK-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%result>, ir<%out>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; TMP2-LABEL: VPlan for loop in 'test'
+; TMP2: VPlan ' for UF>=1' {
+; TMP2-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; TMP2-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; TMP2-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; TMP2-NEXT: Live-in ir<%n> = original trip-count
+; TMP2-EMPTY:
+; TMP2-NEXT: ir-bb<entry>:
+; TMP2-NEXT: Successor(s): scalar.ph, vector.ph
+; TMP2-EMPTY:
+; TMP2-NEXT: vector.ph:
+; TMP2-NEXT: Successor(s): vector loop
+; TMP2-EMPTY:
+; TMP2-NEXT: <x1> vector loop: {
+; TMP2-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; TMP2-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
+; TMP2-EMPTY:
+; TMP2-NEXT: vector.body:
+; TMP2-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; TMP2-NEXT: Successor(s): vector.body.split
+; TMP2-EMPTY:
+; TMP2-NEXT: vector.body.split:
+; TMP2-NEXT: Successor(s): check.cond1
+; TMP2-EMPTY:
+; TMP2-NEXT: check.cond1:
+; TMP2-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; TMP2-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; TMP2-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP6]]>
+; TMP2-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP6]]>
+; TMP2-NEXT: Successor(s): load.v0
+; TMP2-EMPTY:
+; TMP2-NEXT: load.v0:
+; TMP2-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; TMP2-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv>
+; TMP2-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, vp<[[VP7]]>
+; TMP2-NEXT: Successor(s): load.v1
+; TMP2-EMPTY:
+; TMP2-NEXT: load.v1:
+; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%cond1>
+; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP8]]>
+; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP8]]>
+; TMP2-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv>
+; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP9]]>
+; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP9]]>
+; TMP2-NEXT: Successor(s): load.v2.check
+; TMP2-EMPTY:
+; TMP2-NEXT: load.v2.check:
+; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%cond1>
+; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP10]]>
+; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP9]]> ir<0>/vp<[[VP11]]>
+; TMP2-NEXT: Successor(s): load.v2
+; TMP2-EMPTY:
+; TMP2-NEXT: load.v2:
+; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3>
+; TMP2-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv>
+; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP12]]>
+; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP12]]>
+; TMP2-NEXT: Successor(s): latch
+; TMP2-EMPTY:
+; TMP2-NEXT: latch:
+; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3>
+; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]>
+; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP12]]> ir<%val2>/vp<[[VP14]]>
+; TMP2-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv>
+; TMP2-NEXT: EMIT store ir<%result>, ir<%out>, vp<[[VP4]]>
+; TMP2-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>, vp<[[VP4]]>
+; TMP2-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n>, vp<[[VP4]]>
+; TMP2-NEXT: Successor(s): vector.latch
+; TMP2-EMPTY:
+; TMP2-NEXT: vector.latch:
+; TMP2-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; TMP2-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; TMP2-NEXT: No successors
+; TMP2-NEXT: }
+; TMP2-NEXT: Successor(s): middle.block
+; TMP2-EMPTY:
+; TMP2-NEXT: middle.block:
+; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; TMP2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; TMP2-NEXT: EMIT branch-on-cond ir<true>
+; TMP2-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; TMP2-EMPTY:
+; TMP2-NEXT: ir-bb<exit>:
+; TMP2-NEXT: No successors
+; TMP2-EMPTY:
+; TMP2-NEXT: scalar.ph:
+; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; TMP2-NEXT: Successor(s): ir-bb<loop>
+; TMP2-EMPTY:
+; TMP2-NEXT: ir-bb<loop>:
+; TMP2-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; TMP2-NEXT: No successors
+; TMP2-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ br i1 %c1, label %load.v0, label %check.cond1
+
+check.cond1:
+ %not.c2 = xor i1 %c2, true
+ %cond1 = or i1 %c1, %not.c2
+ br i1 %cond1, label %load.v1, label %load.v2.check
+
+load.v0:
+ %gep0 = getelementptr inbounds i32, ptr %src0, i64 %iv
+ %v0 = load i32, ptr %gep0, align 4
+ br label %load.v1
+
+load.v1:
+ %val0 = phi i32 [ %v0, %load.v0 ], [ 0, %check.cond1 ]
+ %gep1 = getelementptr inbounds i32, ptr %src1, i64 %iv
+ %v1 = load i32, ptr %gep1, align 4
+ %val1 = add i32 %v1, %val0
+ br label %load.v2.check
+
+load.v2.check:
+ %val2 = phi i32 [ %val1, %load.v1 ], [ 0, %check.cond1 ]
+ br i1 %c3, label %load.v2, label %latch
+
+load.v2:
+ %gep2 = getelementptr inbounds i32, ptr %src2, i64 %iv
+ %v2 = load i32, ptr %gep2, align 4
+ %val3 = add i32 %v2, %val2
+ br label %latch
+
+latch:
+ %result = phi i32 [ %val3, %load.v2 ], [ %val2, %load.v2.check ]
+ %out = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %result, ptr %out, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 2920ae187a831f0993144d9aecac8ce77ef70d8b Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 08:52:32 -0700
Subject: [PATCH 2/3] [VPlan] Reassociate header mask in the predicator during
mask creation
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 51 +++++++++++++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 29 ---------
.../LoopVectorize/RISCV/low-trip-count.ll | 19 +++---
.../RISCV/tail-folding-complex-mask.ll | 4 +-
.../LoopVectorize/VPlan/predicator.ll | 65 +++++++++++--------
.../VPlan/vplan-print-before-after-all.ll | 2 -
6 files changed, 95 insertions(+), 75 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index c9e1eae12eb99..29c3366f9560f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -56,6 +56,15 @@ class VPPredicator {
/// possibly inserting new recipes at \p Dst (using Builder's insertion point)
VPValue *createEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst);
+ /// Create a logical-and of a source mask and edge condition, keeping the
+ /// header mask outermost when present. E.g. createMaskAnd("H && M", "C")
+ /// would result in "H && (M && C)". The edge condition must not contain the
+ /// header mask.
+ VPValue *createMaskAnd(VPValue *SrcMask, VPValue *EdgeCond, DebugLoc DL);
+
+ /// Create a logical-or, factoring out a common header mask if present.
+ VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL);
+
/// Record \p Mask as the *entry* mask of \p VPBB, which is expected to not
/// already have a mask.
void setBlockInMask(const VPBasicBlock *VPBB, VPValue *Mask) {
@@ -118,6 +127,38 @@ class VPPredicator {
};
} // namespace
+VPValue *VPPredicator::createMaskAnd(VPValue *SrcMask, VPValue *EdgeCond,
+ DebugLoc DL) {
+ VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
+ VPValue *Remainder = nullptr;
+ if (!HeaderMask ||
+ !match(SrcMask, m_RemoveMask(HeaderMask, Remainder)))
+ return Builder.createLogicalAnd(SrcMask, EdgeCond, DL);
+
+ [[maybe_unused]] VPValue *EdgeRemainder = nullptr;
+ assert(!match(EdgeCond, m_RemoveMask(HeaderMask, EdgeRemainder)) &&
+ "Edge condition must not contain the header mask");
+
+ if (!Remainder)
+ return Builder.createLogicalAnd(HeaderMask, EdgeCond, DL);
+ return Builder.createLogicalAnd(
+ HeaderMask, Builder.createLogicalAnd(Remainder, EdgeCond, DL), DL);
+}
+
+VPValue *VPPredicator::createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) {
+ VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
+ VPValue *LHSRemainder = nullptr;
+ VPValue *RHSRemainder = nullptr;
+ if (!HeaderMask || !match(LHS, m_RemoveMask(HeaderMask, LHSRemainder)) ||
+ !match(RHS, m_RemoveMask(HeaderMask, RHSRemainder)))
+ return Builder.createOr(LHS, RHS, DL);
+
+ if (!LHSRemainder || !RHSRemainder)
+ return HeaderMask;
+ return Builder.createLogicalAnd(
+ HeaderMask, Builder.createOr(LHSRemainder, RHSRemainder, DL), DL);
+}
+
VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src,
const VPBasicBlock *Dst) {
assert(is_contained(Dst->getPredecessors(), Src) && "Invalid edge");
@@ -154,7 +195,7 @@ VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src,
// The bitwise 'And' of SrcMask and EdgeMask introduces new UB if SrcMask
// is false and EdgeMask is poison. Avoid that by using 'LogicalAnd'
// instead which generates 'select i1 SrcMask, i1 EdgeMask, i1 false'.
- EdgeMask = Builder.createLogicalAnd(SrcMask, EdgeMask, Term->getDebugLoc());
+ EdgeMask = createMaskAnd(SrcMask, EdgeMask, Term->getDebugLoc());
}
return setEdgeMask(Src, Dst, EdgeMask);
@@ -191,7 +232,7 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
continue;
}
- BlockMask = Builder.createOr(BlockMask, EdgeMask, {});
+ BlockMask = createMaskOr(BlockMask, EdgeMask, {});
}
setBlockInMask(VPBB, BlockMask);
@@ -230,7 +271,7 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) {
for (VPValue *V : drop_begin(Conds))
Mask = Builder.createOr(Mask, V);
if (SrcMask)
- Mask = Builder.createLogicalAnd(SrcMask, Mask);
+ Mask = createMaskAnd(SrcMask, Mask, {});
setEdgeMask(Src, Dst, Mask);
// 2. Create the mask for the default destination, which is reached if
@@ -243,7 +284,7 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) {
if (DefaultMask) {
DefaultMask = Builder.createNot(DefaultMask);
if (SrcMask)
- DefaultMask = Builder.createLogicalAnd(SrcMask, DefaultMask);
+ DefaultMask = createMaskAnd(SrcMask, DefaultMask, {});
} else {
// There are no destinations other than the default destination, so this is
// an unconditional branch.
@@ -346,7 +387,7 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
Builder.setInsertPoint(Dst, getMaskInsertPoint(Dst));
EdgeMask = createEdgeMask(Src, Dst);
}
- Mask = Mask ? Builder.createOr(Mask, EdgeMask) : EdgeMask;
+ Mask = Mask ? createMaskOr(Mask, EdgeMask, {}) : EdgeMask;
}
return Mask;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dd85eeaa8592c..7499a02a5b897 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1661,33 +1661,6 @@ void VPlanTransforms::simplifyReverses(VPlan &Plan) {
R.getVPSingleValue()->replaceAllUsesWith(X);
}
-/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
-/// header mask to be simplified further when tail folding, e.g. in
-/// optimizeEVLMasks.
-static void reassociateHeaderMask(VPlan &Plan) {
- VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
- if (!HeaderMask)
- return;
-
- SmallVector<VPUser *> Worklist;
- for (VPUser *U : HeaderMask->users())
- if (match(U, m_LogicalAnd(m_Specific(HeaderMask), m_VPValue())))
- append_range(Worklist, cast<VPSingleDefRecipe>(U)->users());
-
- while (!Worklist.empty()) {
- auto *R = dyn_cast<VPSingleDefRecipe>(Worklist.pop_back_val());
- VPValue *X, *Y;
- if (!R || !match(R, m_LogicalAnd(
- m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(X)),
- m_VPValue(Y))))
- continue;
- append_range(Worklist, R->users());
- VPBuilder Builder(R);
- R->replaceAllUsesWith(
- Builder.createLogicalAnd(HeaderMask, Builder.createLogicalAnd(X, Y)));
- }
-}
-
static std::optional<Instruction::BinaryOps>
getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
switch (ID) {
@@ -2567,14 +2540,12 @@ bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
void VPlanTransforms::optimize(VPlan &Plan) {
RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
- RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
RUN_VPLAN_PASS(simplifyRecipes, Plan);
RUN_VPLAN_PASS(removeDeadRecipes, Plan);
RUN_VPLAN_PASS(simplifyBlends, Plan);
RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
RUN_VPLAN_PASS(removeRedundantExpandSCEVRecipes, Plan);
- RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
RUN_VPLAN_PASS(simplifyRecipes, Plan);
RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
RUN_VPLAN_PASS(simplifyReverses, Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
index c4e3f51eff4d6..822e67b1b4394 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
@@ -280,18 +280,17 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP12:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
-; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = or <vscale x 4 x i1> [[TMP13]], [[BROADCAST_SPLAT4]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <vscale x 4 x float> splat (float 1.000000e+00), <vscale x 4 x float> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT5]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = or <vscale x 4 x i1> [[TMP13]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C3]], <vscale x 4 x float> splat (float 1.000000e+00), <vscale x 4 x float> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = select <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x float> [[PREDPHI]], <vscale x 4 x float> splat (float 2.000000e+00)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> zeroinitializer, i32 57)
-; CHECK-NEXT: [[TMP10:%.*]] = select <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
-; CHECK-NEXT: [[PREDPHI5:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x float> [[PREDPHI]], <vscale x 4 x float> splat (float 2.000000e+00)
; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[PREDPHI5]], ptr align 4 [[DST:%.*]], <vscale x 4 x i1> splat (i1 true), i32 57)
; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
; CHECK: middle.block:
@@ -360,7 +359,7 @@ define i8 @mul_non_pow_2_low_trip_count(ptr noalias %a) {
; CHECK-NEXT: [[MUL]] = mul i8 [[TMP5]], [[RDX]]
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 10
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: for.end:
; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i8 [ [[MUL]], [[FOR_BODY]] ]
; CHECK-NEXT: ret i8 [[MUL_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 1ffe82b366a38..0792fc784b211 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -21,6 +21,8 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[TMP1:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer
; IF-EVL-NEXT: [[TMP4:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[TMP3]]
+; IF-EVL-NEXT: [[TMP5:%.*]] = xor <vscale x 4 x i1> [[TMP1]], splat (i1 true)
+; IF-EVL-NEXT: [[TMP6:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i1> zeroinitializer
; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0
; IF-EVL-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -34,7 +36,7 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[SRC1]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[VP_OP_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[TMP4]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD7]], [[PREDPHI]]
-; IF-EVL-NEXT: [[PREDPHI8:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP7]])
+; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP12]]
; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> [[BROADCAST_SPLAT4]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[PREDPHI8]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 16f340818345f..a273b778fb2d9 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -1032,22 +1032,28 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1
; TMP1-EMPTY:
; TMP1-NEXT: if.else1:
; TMP1-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2>
-; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP7]]>
-; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
-; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP9]]>
-; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP8]]> ir<1.000000e+00>/vp<[[VP9]]>
+; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP7]]>
+; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]>
+; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP8]]>, ir<%c1>
+; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP11]]>
+; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP9]]> ir<1.000000e+00>/vp<[[VP10]]>
; TMP1-NEXT: Successor(s): if.else2
; TMP1-EMPTY:
; TMP1-NEXT: if.else2:
-; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = not ir<%c3>
-; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP10]]>, vp<[[VP11]]>
-; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%c2>
-; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP13]]>
+; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3>
+; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP11]]>, vp<[[VP13]]>
+; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP14]]>
+; TMP1-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%c2>
+; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]>
+; TMP1-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = or vp<[[VP14]]>, vp<[[VP16]]>
+; TMP1-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP18]]>
; TMP1-NEXT: Successor(s): latch
; TMP1-EMPTY:
; TMP1-NEXT: latch:
-; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP10]]>, ir<%c3>
-; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP14]]> ir<%phi1>/vp<[[VP15]]>
+; TMP1-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%c3>
+; TMP1-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP20]]>
+; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP19]]> ir<%phi1>/vp<[[VP21]]>
; TMP1-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv>
; TMP1-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP4]]>
; TMP1-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>, vp<[[VP4]]>
@@ -1062,7 +1068,7 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1
; TMP1-NEXT: Successor(s): middle.block
; TMP1-EMPTY:
; TMP1-NEXT: middle.block:
-; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; TMP1-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = exiting-iv-value ir<%iv>
; TMP1-NEXT: EMIT vp<%cmp.n> = icmp eq ir<57>, vp<[[VP2]]>
; TMP1-NEXT: EMIT branch-on-cond ir<true>
; TMP1-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1071,7 +1077,7 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1
; TMP1-NEXT: No successors
; TMP1-EMPTY:
; TMP1-NEXT: scalar.ph:
-; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP17]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP23]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; TMP1-NEXT: Successor(s): ir-bb<header>
; TMP1-EMPTY:
; TMP1-NEXT: ir-bb<header>:
@@ -1201,31 +1207,34 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; TMP2-NEXT: Successor(s): load.v1
; TMP2-EMPTY:
; TMP2-NEXT: load.v1:
-; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%cond1>
-; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP8]]>
-; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP8]]>
+; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%cond1>
+; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]>
+; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or ir<%c1>, vp<[[VP8]]>
+; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP10]]>
+; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP9]]>
; TMP2-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv>
-; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP9]]>
-; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP9]]>
+; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP11]]>
+; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP11]]>
; TMP2-NEXT: Successor(s): load.v2.check
; TMP2-EMPTY:
; TMP2-NEXT: load.v2.check:
-; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%cond1>
-; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP10]]>
-; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP9]]> ir<0>/vp<[[VP11]]>
+; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = not ir<%cond1>
+; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP12]]>
+; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]>
+; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP11]]> ir<0>/vp<[[VP14]]>
; TMP2-NEXT: Successor(s): load.v2
; TMP2-EMPTY:
; TMP2-NEXT: load.v2:
-; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3>
+; TMP2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3>
; TMP2-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv>
-; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP12]]>
-; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP12]]>
+; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP15]]>
+; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP15]]>
; TMP2-NEXT: Successor(s): latch
; TMP2-EMPTY:
; TMP2-NEXT: latch:
-; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3>
-; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]>
-; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP12]]> ir<%val2>/vp<[[VP14]]>
+; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = not ir<%c3>
+; TMP2-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]>
+; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP15]]> ir<%val2>/vp<[[VP17]]>
; TMP2-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv>
; TMP2-NEXT: EMIT store ir<%result>, ir<%out>, vp<[[VP4]]>
; TMP2-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>, vp<[[VP4]]>
@@ -1240,7 +1249,7 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; TMP2-NEXT: Successor(s): middle.block
; TMP2-EMPTY:
; TMP2-NEXT: middle.block:
-; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; TMP2-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = exiting-iv-value ir<%iv>
; TMP2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; TMP2-NEXT: EMIT branch-on-cond ir<true>
; TMP2-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1249,7 +1258,7 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; TMP2-NEXT: No successors
; TMP2-EMPTY:
; TMP2-NEXT: scalar.ph:
-; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP19]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; TMP2-NEXT: Successor(s): ir-bb<loop>
; TMP2-EMPTY:
; TMP2-NEXT: ir-bb<loop>:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index c2bd046efae87..21e67e691a795 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -48,14 +48,12 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::truncateToMinimalBitwidths
; CHECK-BEFORE: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeRedundantInductionCasts
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeDeadRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyBlends
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] legalizeAndOptimizeInductions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] narrowToSingleScalarRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeRedundantExpandSCEVRecipes
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeBranchOnConst
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyReverses
>From 245630bde1c5221fe4b6173c2011ffb154653751 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 09:01:41 -0700
Subject: [PATCH 3/3] Remove tests from predicator.ll
---
.../LoopVectorize/VPlan/predicator.ll | 352 ------------------
1 file changed, 352 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index a273b778fb2d9..483ab8ad94b50 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -959,355 +959,3 @@ latch:
exit:
ret void
}
-
-; RUN: llvm-extract --func const_tc_with_predicated_store < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vplan-verify-each -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix TMP1
-define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 {
-; CHECK-LABEL: VPlan for loop in 'const_tc_with_predicated_store'
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: Successor(s): if.then
-; CHECK-EMPTY:
-; CHECK-NEXT: if.then:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
-; CHECK-NEXT: Successor(s): if.else1
-; CHECK-EMPTY:
-; CHECK-NEXT: if.else1:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP6]]>, ir<%c1>
-; CHECK-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP6]]> ir<1.000000e+00>/ir<%c1>
-; CHECK-NEXT: Successor(s): if.else2
-; CHECK-EMPTY:
-; CHECK-NEXT: if.else2:
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c3>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
-; CHECK-NEXT: Successor(s): latch
-; CHECK-EMPTY:
-; CHECK-NEXT: latch:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c3>
-; CHECK-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP11]]> ir<%phi1>/vp<[[VP12]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
-; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-;
-; TMP1-LABEL: VPlan for loop in 'const_tc_with_predicated_store'
-; TMP1: VPlan ' for UF>=1' {
-; TMP1-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; TMP1-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; TMP1-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; TMP1-NEXT: Live-in ir<57> = original trip-count
-; TMP1-EMPTY:
-; TMP1-NEXT: ir-bb<entry>:
-; TMP1-NEXT: Successor(s): scalar.ph, vector.ph
-; TMP1-EMPTY:
-; TMP1-NEXT: vector.ph:
-; TMP1-NEXT: Successor(s): vector loop
-; TMP1-EMPTY:
-; TMP1-NEXT: <x1> vector loop: {
-; TMP1-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
-; TMP1-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
-; TMP1-EMPTY:
-; TMP1-NEXT: vector.body:
-; TMP1-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; TMP1-NEXT: Successor(s): vector.body.split
-; TMP1-EMPTY:
-; TMP1-NEXT: vector.body.split:
-; TMP1-NEXT: Successor(s): if.then
-; TMP1-EMPTY:
-; TMP1-NEXT: if.then:
-; TMP1-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
-; TMP1-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
-; TMP1-NEXT: Successor(s): if.else1
-; TMP1-EMPTY:
-; TMP1-NEXT: if.else1:
-; TMP1-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2>
-; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP7]]>
-; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]>
-; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
-; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP8]]>, ir<%c1>
-; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP11]]>
-; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP9]]> ir<1.000000e+00>/vp<[[VP10]]>
-; TMP1-NEXT: Successor(s): if.else2
-; TMP1-EMPTY:
-; TMP1-NEXT: if.else2:
-; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3>
-; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP11]]>, vp<[[VP13]]>
-; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP14]]>
-; TMP1-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%c2>
-; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]>
-; TMP1-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = or vp<[[VP14]]>, vp<[[VP16]]>
-; TMP1-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP18]]>
-; TMP1-NEXT: Successor(s): latch
-; TMP1-EMPTY:
-; TMP1-NEXT: latch:
-; TMP1-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%c3>
-; TMP1-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP20]]>
-; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP19]]> ir<%phi1>/vp<[[VP21]]>
-; TMP1-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv>
-; TMP1-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP4]]>
-; TMP1-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; TMP1-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56>, vp<[[VP4]]>
-; TMP1-NEXT: Successor(s): vector.latch
-; TMP1-EMPTY:
-; TMP1-NEXT: vector.latch:
-; TMP1-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; TMP1-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; TMP1-NEXT: No successors
-; TMP1-NEXT: }
-; TMP1-NEXT: Successor(s): middle.block
-; TMP1-EMPTY:
-; TMP1-NEXT: middle.block:
-; TMP1-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; TMP1-NEXT: EMIT vp<%cmp.n> = icmp eq ir<57>, vp<[[VP2]]>
-; TMP1-NEXT: EMIT branch-on-cond ir<true>
-; TMP1-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; TMP1-EMPTY:
-; TMP1-NEXT: ir-bb<exit>:
-; TMP1-NEXT: No successors
-; TMP1-EMPTY:
-; TMP1-NEXT: scalar.ph:
-; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP23]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; TMP1-NEXT: Successor(s): ir-bb<header>
-; TMP1-EMPTY:
-; TMP1-NEXT: ir-bb<header>:
-; TMP1-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; TMP1-NEXT: No successors
-; TMP1-NEXT: }
-;
-entry:
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
- br i1 %c1, label %if.else1, label %if.then
-
-if.then:
- br i1 %c2, label %if.else2, label %if.else1
-
-if.else1:
- %phi1 = phi float [ 0.0, %if.then ], [ 1.0, %header ]
- br i1 %c3, label %latch, label %if.else2
-
-if.else2:
- br label %latch
-
-latch:
- %phi = phi float [ %phi1, %if.else1 ], [ 2.0, %if.else2 ]
- %gep = getelementptr float, ptr %dst, i64 %iv
- store float %phi, ptr %gep, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv, 56
- br i1 %ec, label %exit, label %header
-
-exit:
- ret void
-}
-
-; RUN: llvm-extract --func test < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix=TMP2
-define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src2, ptr noalias %dst, i1 %c1, i1 %c2, i1 %c3) {
-; CHECK-LABEL: VPlan for loop in 'test'
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: Successor(s): check.cond1
-; CHECK-EMPTY:
-; CHECK-NEXT: check.cond1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
-; CHECK-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): load.v0
-; CHECK-EMPTY:
-; CHECK-NEXT: load.v0:
-; CHECK-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, ir<%c1>
-; CHECK-NEXT: Successor(s): load.v1
-; CHECK-EMPTY:
-; CHECK-NEXT: load.v1:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%cond1>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or ir<%c1>, vp<[[VP5]]>
-; CHECK-NEXT: BLEND ir<%val0> = ir<%v0>/ir<%c1> ir<0>/vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP6]]>
-; CHECK-NEXT: Successor(s): load.v2.check
-; CHECK-EMPTY:
-; CHECK-NEXT: load.v2.check:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cond1>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP7]]>
-; CHECK-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP6]]> ir<0>/vp<[[VP8]]>
-; CHECK-NEXT: Successor(s): load.v2
-; CHECK-EMPTY:
-; CHECK-NEXT: load.v2:
-; CHECK-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, ir<%c3>
-; CHECK-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, ir<%c3>
-; CHECK-NEXT: Successor(s): latch
-; CHECK-EMPTY:
-; CHECK-NEXT: latch:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%c3>
-; CHECK-NEXT: BLEND ir<%result> = ir<%val3>/ir<%c3> ir<%val2>/vp<[[VP9]]>
-; CHECK-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%result>, ir<%out>
-; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-;
-; TMP2-LABEL: VPlan for loop in 'test'
-; TMP2: VPlan ' for UF>=1' {
-; TMP2-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; TMP2-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; TMP2-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; TMP2-NEXT: Live-in ir<%n> = original trip-count
-; TMP2-EMPTY:
-; TMP2-NEXT: ir-bb<entry>:
-; TMP2-NEXT: Successor(s): scalar.ph, vector.ph
-; TMP2-EMPTY:
-; TMP2-NEXT: vector.ph:
-; TMP2-NEXT: Successor(s): vector loop
-; TMP2-EMPTY:
-; TMP2-NEXT: <x1> vector loop: {
-; TMP2-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
-; TMP2-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
-; TMP2-EMPTY:
-; TMP2-NEXT: vector.body:
-; TMP2-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; TMP2-NEXT: Successor(s): vector.body.split
-; TMP2-EMPTY:
-; TMP2-NEXT: vector.body.split:
-; TMP2-NEXT: Successor(s): check.cond1
-; TMP2-EMPTY:
-; TMP2-NEXT: check.cond1:
-; TMP2-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
-; TMP2-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
-; TMP2-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP6]]>
-; TMP2-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP6]]>
-; TMP2-NEXT: Successor(s): load.v0
-; TMP2-EMPTY:
-; TMP2-NEXT: load.v0:
-; TMP2-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
-; TMP2-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv>
-; TMP2-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, vp<[[VP7]]>
-; TMP2-NEXT: Successor(s): load.v1
-; TMP2-EMPTY:
-; TMP2-NEXT: load.v1:
-; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%cond1>
-; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]>
-; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or ir<%c1>, vp<[[VP8]]>
-; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP10]]>
-; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP9]]>
-; TMP2-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv>
-; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP11]]>
-; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP11]]>
-; TMP2-NEXT: Successor(s): load.v2.check
-; TMP2-EMPTY:
-; TMP2-NEXT: load.v2.check:
-; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = not ir<%cond1>
-; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP12]]>
-; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]>
-; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP11]]> ir<0>/vp<[[VP14]]>
-; TMP2-NEXT: Successor(s): load.v2
-; TMP2-EMPTY:
-; TMP2-NEXT: load.v2:
-; TMP2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3>
-; TMP2-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv>
-; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP15]]>
-; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP15]]>
-; TMP2-NEXT: Successor(s): latch
-; TMP2-EMPTY:
-; TMP2-NEXT: latch:
-; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = not ir<%c3>
-; TMP2-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]>
-; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP15]]> ir<%val2>/vp<[[VP17]]>
-; TMP2-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv>
-; TMP2-NEXT: EMIT store ir<%result>, ir<%out>, vp<[[VP4]]>
-; TMP2-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>, vp<[[VP4]]>
-; TMP2-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n>, vp<[[VP4]]>
-; TMP2-NEXT: Successor(s): vector.latch
-; TMP2-EMPTY:
-; TMP2-NEXT: vector.latch:
-; TMP2-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; TMP2-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; TMP2-NEXT: No successors
-; TMP2-NEXT: }
-; TMP2-NEXT: Successor(s): middle.block
-; TMP2-EMPTY:
-; TMP2-NEXT: middle.block:
-; TMP2-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; TMP2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; TMP2-NEXT: EMIT branch-on-cond ir<true>
-; TMP2-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; TMP2-EMPTY:
-; TMP2-NEXT: ir-bb<exit>:
-; TMP2-NEXT: No successors
-; TMP2-EMPTY:
-; TMP2-NEXT: scalar.ph:
-; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP19]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; TMP2-NEXT: Successor(s): ir-bb<loop>
-; TMP2-EMPTY:
-; TMP2-NEXT: ir-bb<loop>:
-; TMP2-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; TMP2-NEXT: No successors
-; TMP2-NEXT: }
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
- br i1 %c1, label %load.v0, label %check.cond1
-
-check.cond1:
- %not.c2 = xor i1 %c2, true
- %cond1 = or i1 %c1, %not.c2
- br i1 %cond1, label %load.v1, label %load.v2.check
-
-load.v0:
- %gep0 = getelementptr inbounds i32, ptr %src0, i64 %iv
- %v0 = load i32, ptr %gep0, align 4
- br label %load.v1
-
-load.v1:
- %val0 = phi i32 [ %v0, %load.v0 ], [ 0, %check.cond1 ]
- %gep1 = getelementptr inbounds i32, ptr %src1, i64 %iv
- %v1 = load i32, ptr %gep1, align 4
- %val1 = add i32 %v1, %val0
- br label %load.v2.check
-
-load.v2.check:
- %val2 = phi i32 [ %val1, %load.v1 ], [ 0, %check.cond1 ]
- br i1 %c3, label %load.v2, label %latch
-
-load.v2:
- %gep2 = getelementptr inbounds i32, ptr %src2, i64 %iv
- %v2 = load i32, ptr %gep2, align 4
- %val3 = add i32 %v2, %val2
- br label %latch
-
-latch:
- %result = phi i32 [ %val3, %load.v2 ], [ %val2, %load.v2.check ]
- %out = getelementptr inbounds i32, ptr %dst, i64 %iv
- store i32 %result, ptr %out, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- ret void
-}
More information about the llvm-branch-commits
mailing list