[llvm] 3c5e0d8 - [LoopVectorize] Clear cache of `LoopAccessInfoManager`
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 11 01:04:04 PST 2023
Author: Miguel Saldivar
Date: 2023-01-11T09:03:40Z
New Revision: 3c5e0d87f80efa8068a58768514d916d9dc03f3d
URL: https://github.com/llvm/llvm-project/commit/3c5e0d87f80efa8068a58768514d916d9dc03f3d
DIFF: https://github.com/llvm/llvm-project/commit/3c5e0d87f80efa8068a58768514d916d9dc03f3d.diff
LOG: [LoopVectorize] Clear cache of `LoopAccessInfoManager`
LAI is cached during the LoopDistribute pass, and is later re-used during LoopVectorize. The problem is that LoopVectorize changes SCEV, and the cached LAI does not get updated. Hence, when re-using the cached LAI, it references an invalid SCEV.
Fixes #59319
Reviewed By: fhahn
Differential Revision: https://reviews.llvm.org/D139601
Added:
llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6f8550b24c415..ecfdae6257d31 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10633,6 +10633,9 @@ LoopVectorizeResult LoopVectorizePass::runImpl(
Changed |= formLCSSARecursively(*L, *DT, LI, SE);
Changed |= CFGChanged |= processLoop(L);
+
+ if (Changed)
+ LAIs->clear();
}
// Process each loop nest in the function.
diff --git a/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
new file mode 100644
index 0000000000000..e7af6137ad11c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-distribute,loop-vectorize -enable-loop-distribute -force-vector-width=4 -force-vector-interleave=1 -S \
+; RUN: %s | FileCheck %s
+
+; This test is to assure LoopAccessInfo invalidation after LoopVectorize
+; modifies the IR.
+define void @reduced(i32* %0, i32* %1, i64 %iv, i32* %2, i64 %iv76, i64 %iv93) {
+; CHECK-LABEL: @reduced(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[TMP1:%.*]] to i8*
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[IV:%.*]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
+; CHECK-NEXT: [[IND_END:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[IND_END]]
+; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[IND_END]]
+; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i64 [[IND_END]], 1
+; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOP_2_PREHEADER:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT: br label [[LOOP_1:%.*]]
+; CHECK: loop.1:
+; CHECK-NEXT: [[IV761:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT77:%.*]], [[LOOP_1]] ]
+; CHECK-NEXT: [[IV4:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_1]] ]
+; CHECK-NEXT: [[IV_NEXT77]] = add i64 [[IV761]], 1
+; CHECK-NEXT: [[ARRAYIDX_I_I50:%.*]] = getelementptr i32, i32* [[TMP0:%.*]], i64 [[IV76:%.*]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV4]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV4]], [[IV]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[LOOP_2_PREHEADER]], label [[LOOP_1]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: loop.2.preheader:
+; CHECK-NEXT: [[IV761_LCSSA:%.*]] = phi i64 [ [[IV761]], [[LOOP_1]] ], [ [[IND_ESCAPE]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MIN_ITERS_CHECK9:%.*]] = icmp ult i64 [[TMP4]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK9]], label [[SCALAR_PH8:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[TMP1]], i64 1
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8*
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[IV761_LCSSA]]
+; CHECK-NEXT: [[SCEVGEP34:%.*]] = bitcast i32* [[SCEVGEP3]] to i8*
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[IV761_LCSSA]], 1
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[TMP10]]
+; CHECK-NEXT: [[SCEVGEP56:%.*]] = bitcast i32* [[SCEVGEP5]] to i8*
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[TMP3]], [[SCEVGEP56]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH8]], label [[VECTOR_PH10:%.*]]
+; CHECK: vector.ph10:
+; CHECK-NEXT: [[N_MOD_VF11:%.*]] = urem i64 [[TMP4]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF11]]
+; CHECK-NEXT: br label [[VECTOR_BODY14:%.*]]
+; CHECK: vector.body14:
+; CHECK-NEXT: [[INDEX15:%.*]] = phi i64 [ 0, [[VECTOR_PH10]] ], [ [[INDEX_NEXT16:%.*]], [[VECTOR_BODY14]] ]
+; CHECK-NEXT: store i32 0, i32* [[TMP1]], align 4, !alias.scope !4, !noalias !7
+; CHECK-NEXT: [[INDEX_NEXT16]] = add nuw i64 [[INDEX15]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT16]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK7:%.*]], label [[VECTOR_BODY14]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: middle.block7:
+; CHECK-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N13]], label [[LOOP_3_LR_PH:%.*]], label [[SCALAR_PH8]]
+; CHECK: scalar.ph8:
+; CHECK-NEXT: [[BC_RESUME_VAL12:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK7]] ], [ 0, [[LOOP_2_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label [[LOOP_2:%.*]]
+; CHECK: loop.3.lr.ph:
+; CHECK-NEXT: [[IDXPROM_I_I61:%.*]] = and i64 [[IV761_LCSSA]], 1
+; CHECK-NEXT: [[ARRAYIDX_I_I62:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[IDXPROM_I_I61]]
+; CHECK-NEXT: [[ARRAYIDX_I_I6220:%.*]] = bitcast i32* [[ARRAYIDX_I_I62]] to i8*
+; CHECK-NEXT: [[MIN_ITERS_CHECK28:%.*]] = icmp ult i64 [[TMP4]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK28]], label [[SCALAR_PH27:%.*]], label [[VECTOR_MEMCHECK17:%.*]]
+; CHECK: vector.memcheck17:
+; CHECK-NEXT: [[SCEVGEP18:%.*]] = getelementptr i32, i32* [[TMP1]], i64 1
+; CHECK-NEXT: [[SCEVGEP1819:%.*]] = bitcast i32* [[SCEVGEP18]] to i8*
+; CHECK-NEXT: [[TMP12:%.*]] = add nuw nsw i64 [[IDXPROM_I_I61]], 1
+; CHECK-NEXT: [[SCEVGEP21:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[TMP12]]
+; CHECK-NEXT: [[SCEVGEP2122:%.*]] = bitcast i32* [[SCEVGEP21]] to i8*
+; CHECK-NEXT: [[BOUND023:%.*]] = icmp ult i8* [[TMP3]], [[SCEVGEP2122]]
+; CHECK-NEXT: [[BOUND124:%.*]] = icmp ult i8* [[ARRAYIDX_I_I6220]], [[SCEVGEP1819]]
+; CHECK-NEXT: [[FOUND_CONFLICT25:%.*]] = and i1 [[BOUND023]], [[BOUND124]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT25]], label [[SCALAR_PH27]], label [[VECTOR_PH29:%.*]]
+; CHECK: vector.ph29:
+; CHECK-NEXT: [[N_MOD_VF30:%.*]] = urem i64 [[TMP4]], 4
+; CHECK-NEXT: [[N_VEC31:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF30]]
+; CHECK-NEXT: br label [[VECTOR_BODY34:%.*]]
+; CHECK: vector.body34:
+; CHECK-NEXT: [[INDEX35:%.*]] = phi i64 [ 0, [[VECTOR_PH29]] ], [ [[INDEX_NEXT36:%.*]], [[VECTOR_BODY34]] ]
+; CHECK-NEXT: store i32 0, i32* [[TMP1]], align 4, !alias.scope !10, !noalias !13
+; CHECK-NEXT: [[INDEX_NEXT36]] = add nuw i64 [[INDEX35]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT36]], [[N_VEC31]]
+; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK26:%.*]], label [[VECTOR_BODY34]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: middle.block26:
+; CHECK-NEXT: [[CMP_N33:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC31]]
+; CHECK-NEXT: br i1 [[CMP_N33]], label [[LOOP_CLEANUP:%.*]], label [[SCALAR_PH27]]
+; CHECK: scalar.ph27:
+; CHECK-NEXT: [[BC_RESUME_VAL32:%.*]] = phi i64 [ [[N_VEC31]], [[MIDDLE_BLOCK26]] ], [ 0, [[LOOP_3_LR_PH]] ], [ 0, [[VECTOR_MEMCHECK17]] ]
+; CHECK-NEXT: br label [[LOOP_3:%.*]]
+; CHECK: loop.2:
+; CHECK-NEXT: [[IV846:%.*]] = phi i64 [ [[IV_NEXT85:%.*]], [[LOOP_2]] ], [ [[BC_RESUME_VAL12]], [[SCALAR_PH8]] ]
+; CHECK-NEXT: [[IV_NEXT87:%.*]] = add i64 0, 0
+; CHECK-NEXT: [[ARRAYIDX_I_I56:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[IV761_LCSSA]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i32, i32* [[ARRAYIDX_I_I56]], align 4
+; CHECK-NEXT: store i32 0, i32* [[TMP1]], align 4
+; CHECK-NEXT: [[IV_NEXT85]] = add i64 [[IV846]], 1
+; CHECK-NEXT: [[EXITCOND92_NOT:%.*]] = icmp eq i64 [[IV846]], [[IV]]
+; CHECK-NEXT: br i1 [[EXITCOND92_NOT]], label [[LOOP_3_LR_PH]], label [[LOOP_2]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: loop.3:
+; CHECK-NEXT: [[IV932:%.*]] = phi i64 [ [[BC_RESUME_VAL32]], [[SCALAR_PH27]] ], [ [[IV_NEXT94:%.*]], [[LOOP_3]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, i32* [[ARRAYIDX_I_I62]], align 4
+; CHECK-NEXT: [[ARRAYIDX_I_I653:%.*]] = getelementptr i32, i32* [[TMP2:%.*]], i64 [[IV93:%.*]]
+; CHECK-NEXT: store i32 0, i32* [[TMP1]], align 4
+; CHECK-NEXT: [[IV_NEXT94]] = add i64 [[IV932]], 1
+; CHECK-NEXT: [[EXITCOND97_NOT:%.*]] = icmp eq i64 [[IV932]], [[IV]]
+; CHECK-NEXT: br i1 [[EXITCOND97_NOT]], label [[LOOP_CLEANUP]], label [[LOOP_3]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: loop.cleanup:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop.1
+
+loop.1: ; preds = %loop.1, %entry
+ %iv761 = phi i64 [ 0, %entry ], [ %iv.next77, %loop.1 ]
+ %iv4 = phi i64 [ 0, %entry ], [ %iv.next, %loop.1 ]
+ %iv.next77 = add i64 %iv761, 1
+ %arrayidx.i.i50 = getelementptr i32, i32* %0, i64 %iv76
+ %iv.next = add i64 %iv4, 1
+ %exitcond.not = icmp eq i64 %iv4, %iv
+ br i1 %exitcond.not, label %loop.2.preheader, label %loop.1
+
+loop.2.preheader: ; preds = %loop.1
+ br label %loop.2
+
+loop.3.lr.ph: ; preds = %loop.2
+ %idxprom.i.i61 = and i64 %iv761, 1
+ %arrayidx.i.i62 = getelementptr i32, i32* %0, i64 %idxprom.i.i61
+ br label %loop.3
+
+loop.2: ; preds = %loop.2, %loop.2.preheader
+ %iv846 = phi i64 [ %iv.next85, %loop.2 ], [ 0, %loop.2.preheader ]
+ %iv.next87 = add i64 0, 0
+ %arrayidx.i.i56 = getelementptr i32, i32* %0, i64 %iv761
+ %3 = load i32, i32* %arrayidx.i.i56, align 4
+ store i32 0, i32* %1, align 4
+ %iv.next85 = add i64 %iv846, 1
+ %exitcond92.not = icmp eq i64 %iv846, %iv
+ br i1 %exitcond92.not, label %loop.3.lr.ph, label %loop.2
+
+loop.3: ; preds = %loop.3, %loop.3.lr.ph
+ %iv932 = phi i64 [ 0, %loop.3.lr.ph ], [ %iv.next94, %loop.3 ]
+ %4 = load i32, i32* %arrayidx.i.i62, align 4
+ %arrayidx.i.i653 = getelementptr i32, i32* %2, i64 %iv93
+ store i32 0, i32* %1, align 4
+ %iv.next94 = add i64 %iv932, 1
+ %exitcond97.not = icmp eq i64 %iv932, %iv
+ br i1 %exitcond97.not, label %loop.cleanup, label %loop.3
+
+loop.cleanup: ; preds = %loop.3
+ ret void
+}
More information about the llvm-commits
mailing list