[llvm] [LV] Invalidate SCEV values in the scalar loop after loop vectorization (PR #69886)
Yingwei Zheng via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 23 02:37:40 PDT 2023
https://github.com/dtcxzyw updated https://github.com/llvm/llvm-project/pull/69886
>From 72b287aca006db378d0d8c81c0401edd9a6563d5 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333 at gmail.com>
Date: Mon, 23 Oct 2023 01:08:56 +0800
Subject: [PATCH 1/2] [LV] Invalidate SCEV values in the scalar loop after loop
vectorization
---
.../Transforms/Vectorize/LoopVectorize.cpp | 5 ++-
llvm/test/Transforms/LoopVectorize/pr66616.ll | 35 +++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/pr66616.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a5d523894bef155..8c8b4a1e2c32f7a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3543,7 +3543,10 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
// Forget the original basic block.
PSE.getSE()->forgetLoop(OrigLoop);
- PSE.getSE()->forgetBlockAndLoopDispositions();
+ // Now the scalar loop doesn't dominate any of the loop exits.
+ for (auto *Block : OrigLoop->getBlocks())
+ for (auto &Inst : *Block)
+ PSE.getSE()->forgetValue(&Inst);
// After vectorization, the exit blocks of the original loop will have
// additional predecessors. Invalidate SCEVs for the exit phis in case SE
diff --git a/llvm/test/Transforms/LoopVectorize/pr66616.ll b/llvm/test/Transforms/LoopVectorize/pr66616.ll
new file mode 100644
index 000000000000000..8374cfacfb71ecc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr66616.ll
@@ -0,0 +1,35 @@
+; RUN: opt -passes="require<scalar-evolution>,print<scalar-evolution>,loop-vectorize" --verify-scev --verify-scev-strict -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @pr66616(
+; CHECK: vector.body
+define void @pr66616() {
+entry:
+ br label %for.body
+
+for.cond5.preheader: ; preds = %for.body
+ br label %while.body.i
+
+for.body: ; preds = %for.body, %entry
+ %storemerge12 = phi i8 [ 0, %entry ], [ %inc, %for.body ]
+ %0 = load i8, ptr null, align 1
+ %conv2 = sext i8 %0 to i32
+ %add3 = add i32 %conv2, 1
+ %inc = add i8 %storemerge12, 1
+ %conv1 = zext i8 %inc to i32
+ %tobool.not = icmp eq i32 %conv1, 0
+ br i1 %tobool.not, label %for.cond5.preheader, label %for.body
+
+while.body.i: ; preds = %while.body.i, %for.cond5.preheader
+ %i.addr.09.i = phi i32 [ %dec.i, %while.body.i ], [ %add3, %for.cond5.preheader ]
+ %incdec.ptr48.i = phi ptr [ %incdec.ptr.i, %while.body.i ], [ null, %for.cond5.preheader ]
+ %dec.i = add i32 %i.addr.09.i, 1
+ %incdec.ptr.i = getelementptr i8, ptr %incdec.ptr48.i, i64 1
+ %tobool.not.i = icmp eq i32 %i.addr.09.i, 0
+ br i1 %tobool.not.i, label %for.inc9.loopexit, label %while.body.i
+
+for.inc9.loopexit: ; preds = %while.body.i
+ ret void
+}
>From 56f32147ae3e4e8752d4f6cb49915bf9a5ac4e92 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333 at gmail.com>
Date: Mon, 23 Oct 2023 17:36:03 +0800
Subject: [PATCH 2/2] fixup! [LV] Invalidate SCEV values in the scalar loop
after loop vectorization
Simplify tests.
---
llvm/test/Transforms/LoopVectorize/pr66616.ll | 35 -------------------
.../LoopVectorize/scev-invalidation.ll | 31 +++++++++++++++-
2 files changed, 30 insertions(+), 36 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/pr66616.ll
diff --git a/llvm/test/Transforms/LoopVectorize/pr66616.ll b/llvm/test/Transforms/LoopVectorize/pr66616.ll
deleted file mode 100644
index 8374cfacfb71ecc..000000000000000
--- a/llvm/test/Transforms/LoopVectorize/pr66616.ll
+++ /dev/null
@@ -1,35 +0,0 @@
-; RUN: opt -passes="require<scalar-evolution>,print<scalar-evolution>,loop-vectorize" --verify-scev --verify-scev-strict -S < %s | FileCheck %s
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK-LABEL: @pr66616(
-; CHECK: vector.body
-define void @pr66616() {
-entry:
- br label %for.body
-
-for.cond5.preheader: ; preds = %for.body
- br label %while.body.i
-
-for.body: ; preds = %for.body, %entry
- %storemerge12 = phi i8 [ 0, %entry ], [ %inc, %for.body ]
- %0 = load i8, ptr null, align 1
- %conv2 = sext i8 %0 to i32
- %add3 = add i32 %conv2, 1
- %inc = add i8 %storemerge12, 1
- %conv1 = zext i8 %inc to i32
- %tobool.not = icmp eq i32 %conv1, 0
- br i1 %tobool.not, label %for.cond5.preheader, label %for.body
-
-while.body.i: ; preds = %while.body.i, %for.cond5.preheader
- %i.addr.09.i = phi i32 [ %dec.i, %while.body.i ], [ %add3, %for.cond5.preheader ]
- %incdec.ptr48.i = phi ptr [ %incdec.ptr.i, %while.body.i ], [ null, %for.cond5.preheader ]
- %dec.i = add i32 %i.addr.09.i, 1
- %incdec.ptr.i = getelementptr i8, ptr %incdec.ptr48.i, i64 1
- %tobool.not.i = icmp eq i32 %i.addr.09.i, 0
- br i1 %tobool.not.i, label %for.inc9.loopexit, label %while.body.i
-
-for.inc9.loopexit: ; preds = %while.body.i
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/scev-invalidation.ll b/llvm/test/Transforms/LoopVectorize/scev-invalidation.ll
index 08163293c14e85e..a85cfef592f8987 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-invalidation.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes="require<scalar-evolution>,print<scalar-evolution>,loop-vectorize" --verify-scev -force-vector-interleave=2 -force-vector-width=8 -S | FileCheck %s
+; RUN: opt < %s -passes="print<scalar-evolution>,loop-vectorize" --verify-scev -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
; CHECK-LABEL: @main(
; CHECK: vector.body
@@ -20,3 +20,32 @@ for.inc16:
%add12 = add i32 %0, %1
br label %for.inc16
}
+
+; CHECK-LABEL: @pr66616(
+; CHECK: vector.body
+define void @pr66616(ptr %ptr) {
+entry:
+ br label %loop.1
+
+loop.1:
+ %iv.1 = phi i8 [ 0, %entry ], [ %inc, %loop.1 ]
+ %load = load i32, ptr %ptr, align 4
+ %add3 = add i32 %load, 1
+ %inc = add i8 %iv.1, 1
+ %cond1 = icmp eq i8 %inc, 0
+ br i1 %cond1, label %preheader, label %loop.1
+
+preheader:
+ br label %loop.2
+
+loop.2:
+ %iv.2 = phi i32 [ %iv.2.i, %loop.2 ], [ %add3, %preheader ]
+ %iv.3 = phi ptr [ %iv.3.i, %loop.2 ], [ %ptr, %preheader ]
+ %iv.2.i = add i32 %iv.2, 1
+ %iv.3.i = getelementptr i8, ptr %iv.3, i64 1
+ %cond2 = icmp eq i32 %iv.2, 0
+ br i1 %cond2, label %exit, label %loop.2
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list