[llvm] [LoopVectorizer] Propagate underlying instruction to the cloned instances of VPPartialReductionRecipes (PR #123638)
Nicholas Guy via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 21 03:41:19 PST 2025
https://github.com/NickGuy-Arm updated https://github.com/llvm/llvm-project/pull/123638
>From 56d61590a6381493c0e914404bee88ea8c714a72 Mon Sep 17 00:00:00 2001
From: Nick Guy <nicholas.guy at arm.com>
Date: Tue, 21 Jan 2025 11:36:28 +0000
Subject: [PATCH 1/2] Test precommit
---
.../AArch64/vplan-epilogue-printing.ll | 94 +++++++++++++++++++
1 file changed, 94 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll
new file mode 100644
index 00000000000000..eef9b68590ef96
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll
@@ -0,0 +1,94 @@
+; REQUIRES: asserts
+; RUN: opt -mattr=+neon,+dotprod -passes=loop-vectorize -debug-only=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 -disable-output %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+; Tests for printing VPlans that are enabled under AArch64
+; CHECK: VPlan 'Final VPlan for VF={8,16},UF={1}' {
+; CHECK-NEXT: Live-in ir<16> = VF * UF
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.main.loop.iter.check>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>:
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: SCALAR-PHI vp<%0> = phi ir<0>, vp<%index.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, vp<%4> (VF scaled by 1/4)
+; CHECK-NEXT: vp<%1> = SCALAR-STEPS vp<%0>, ir<1>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%1>
+; CHECK-NEXT: vp<%2> = vector-pointer ir<%gep.a>
+; CHECK-NEXT: WIDEN ir<%load.a> = load vp<%2>
+; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%1>
+; CHECK-NEXT: vp<%3> = vector-pointer ir<%gep.b>
+; CHECK-NEXT: WIDEN ir<%load.b> = load vp<%3>
+; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
+; CHECK-NEXT: PARTIAL-REDUCE vp<%4> = add ir<%mul>, ir<%accum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%0>, ir<16>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, ir<1024>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: EMIT vp<%6> = compute-reduction-result ir<%accum>, vp<%4>
+; CHECK-NEXT: EMIT vp<%7> = extract-from-end vp<%6>, ir<1>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1024>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<%7> from ir-bb<middle.block>)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<scalar.ph>:
+; CHECK-NEXT: EMIT vp<%vec.epilog.resume.val> = resume-phi ir<1024>, ir<0>
+; CHECK-NEXT: EMIT vp<%bc.merge.rdx> = resume-phi vp<%6>, ir<0>
+; CHECK-NEXT: Successor(s): ir-bb<for.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body>:
+; CHECK-NEXT: IR %accum = phi i32 [ 0, %scalar.ph ], [ %add, %for.body ] (extra operand: vp<%bc.merge.rdx> from ir-bb<scalar.ph>)
+; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv
+; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1
+; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i32
+; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv
+; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1
+; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32
+; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a
+; CHECK-NEXT: IR %add = add i32 %mul, %accum
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+define i32 @print_partial_reduction(ptr %a, ptr %b) {
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ %gep.a = getelementptr i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %ext.a = zext i8 %load.a to i32
+ %gep.b = getelementptr i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul i32 %ext.b, %ext.a
+ %add = add i32 %mul, %accum
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %add
+}
>From 621cd7bfe3f75fcd905a53efb474f3b49e8f6ec9 Mon Sep 17 00:00:00 2001
From: Nick Guy <nicholas.guy at arm.com>
Date: Mon, 20 Jan 2025 16:01:18 +0000
Subject: [PATCH 2/2] [LoopVectorizer] Propagate underlying instruction to the
cloned instances of VPPartialReductionRecipes
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++-
.../LoopVectorize/AArch64/vplan-epilogue-printing.ll | 12 ++++++------
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 87f87bf1437196..33289872010aca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2459,7 +2459,8 @@ class VPPartialReductionRecipe : public VPSingleDefRecipe {
~VPPartialReductionRecipe() override = default;
VPPartialReductionRecipe *clone() override {
- return new VPPartialReductionRecipe(Opcode, getOperand(0), getOperand(1));
+ return new VPPartialReductionRecipe(Opcode, getOperand(0), getOperand(1),
+ getUnderlyingInstr());
}
VP_CLASSOF_IMPL(VPDef::VPPartialReductionSC)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll
index eef9b68590ef96..6810a6bc80adb3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-epilogue-printing.ll
@@ -22,7 +22,7 @@ target triple = "aarch64-none-unknown-elf"
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: SCALAR-PHI vp<%0> = phi ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, vp<%4> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, ir<%add> (VF scaled by 1/4)
; CHECK-NEXT: vp<%1> = SCALAR-STEPS vp<%0>, ir<1>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%1>
; CHECK-NEXT: vp<%2> = vector-pointer ir<%gep.a>
@@ -33,7 +33,7 @@ target triple = "aarch64-none-unknown-elf"
; CHECK-NEXT: WIDEN ir<%load.b> = load vp<%3>
; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
-; CHECK-NEXT: PARTIAL-REDUCE vp<%4> = add ir<%mul>, ir<%accum>
+; CHECK-NEXT: PARTIAL-REDUCE ir<%add> = add ir<%mul>, ir<%accum>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%0>, ir<16>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, ir<1024>
; CHECK-NEXT: No successors
@@ -41,19 +41,19 @@ target triple = "aarch64-none-unknown-elf"
; CHECK-NEXT: Successor(s): ir-bb<middle.block>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<middle.block>:
-; CHECK-NEXT: EMIT vp<%6> = compute-reduction-result ir<%accum>, vp<%4>
-; CHECK-NEXT: EMIT vp<%7> = extract-from-end vp<%6>, ir<1>
+; CHECK-NEXT: EMIT vp<%5> = compute-reduction-result ir<%accum>, ir<%add>
+; CHECK-NEXT: EMIT vp<%6> = extract-from-end vp<%5>, ir<1>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1024>, ir<1024>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<%7> from ir-bb<middle.block>)
+; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<%6> from ir-bb<middle.block>)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<scalar.ph>:
; CHECK-NEXT: EMIT vp<%vec.epilog.resume.val> = resume-phi ir<1024>, ir<0>
-; CHECK-NEXT: EMIT vp<%bc.merge.rdx> = resume-phi vp<%6>, ir<0>
+; CHECK-NEXT: EMIT vp<%bc.merge.rdx> = resume-phi vp<%5>, ir<0>
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
More information about the llvm-commits
mailing list