[llvm] 623c4a7 - [LoopVersioning] Invalidate SCEV for phi if new values are added.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 23 03:53:57 PDT 2022
Author: Florian Hahn
Date: 2022-09-23T11:53:29+01:00
New Revision: 623c4a7a55f716b96070a5c2f83fe0096cb38d38
URL: https://github.com/llvm/llvm-project/commit/623c4a7a55f716b96070a5c2f83fe0096cb38d38
DIFF: https://github.com/llvm/llvm-project/commit/623c4a7a55f716b96070a5c2f83fe0096cb38d38.diff
LOG: [LoopVersioning] Invalidate SCEV for phi if new values are added.
After 20d798bd47ec5191d, SCEV looks through PHIs with a single incoming
value. This means adding a new incoming value may change the SCEV for a
phi. Add missing invalidation when an existing PHI is reused during
LoopVersioning. New incoming values will be added later from the
versioned loop.
Similar issues have been fixed by also adding missing invalidation.
Fixes #57825.
Note that the test case unfortunately requires running loop-vectorize
followed by loop-load-elimination, which does the actual versioning. I
don't think it is possible to reproduce the failure without that
combination.
Added:
llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
Modified:
llvm/lib/Transforms/Utils/LoopVersioning.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index 97f29527bb95c..6309eed7963d1 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -137,8 +137,10 @@ void LoopVersioning::addPHINodes(
// See if we have a single-operand PHI with the value defined by the
// original loop.
for (auto I = PHIBlock->begin(); (PN = dyn_cast<PHINode>(I)); ++I) {
- if (PN->getIncomingValue(0) == Inst)
+ if (PN->getIncomingValue(0) == Inst) {
+ SE->forgetValue(PN);
break;
+ }
}
// If not create it.
if (!PN) {
diff --git a/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll b/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
new file mode 100644
index 0000000000000..4b0bc908eddfa
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -force-vector-width=4 -force-vector-interleave=1 -passes='loop-vectorize,loop-load-elim' -S %s | FileCheck %s
+
+ at glob.1 = external global [100 x double]
+ at glob.2 = external global [100 x double]
+
+define void @g(ptr %dst.1, ptr %start, i64 %N) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT: loop.1.lver.check:
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N:%.*]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @glob.2, i64 [[TMP1]]
+; CHECK-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[DST_1:%.*]], i64 8
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr @glob.2, [[UGLYGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST_1]], [[UGLYGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[LOOP_1_PH_LVER_ORIG:%.*]], label [[LOOP_1_PH:%.*]]
+; CHECK: loop.1.ph.lver.orig:
+; CHECK-NEXT: br label [[LOOP_1_LVER_ORIG:%.*]]
+; CHECK: loop.1.lver.orig:
+; CHECK-NEXT: [[IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[LOOP_1_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], [[LOOP_1_LVER_ORIG]] ]
+; CHECK-NEXT: [[PTR_IV_1_LVER_ORIG:%.*]] = phi ptr [ @glob.1, [[LOOP_1_PH_LVER_ORIG]] ], [ [[PTR_IV_1_NEXT_LVER_ORIG:%.*]], [[LOOP_1_LVER_ORIG]] ]
+; CHECK-NEXT: [[GEP_IV_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr @glob.2, i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT: [[L_1_LVER_ORIG:%.*]] = load double, ptr [[GEP_IV_LVER_ORIG]], align 8
+; CHECK-NEXT: [[GEP_IV_1_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_IV_1_LVER_ORIG]], align 8
+; CHECK-NEXT: store double 0.000000e+00, ptr [[DST_1]], align 8
+; CHECK-NEXT: [[PTR_IV_1_NEXT_LVER_ORIG]] = getelementptr inbounds double, ptr [[PTR_IV_1_LVER_ORIG]], i64 1
+; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[EXITCOND_NOT_LVER_ORIG:%.*]] = icmp eq i64 [[IV_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT_LVER_ORIG]], label [[LOOP_2_PH_LOOPEXIT:%.*]], label [[LOOP_1_LVER_ORIG]]
+; CHECK: loop.1.ph:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load double, ptr @glob.2, align 8
+; CHECK-NEXT: br label [[LOOP_1:%.*]]
+; CHECK: loop.1:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi double [ [[LOAD_INITIAL]], [[LOOP_1_PH]] ], [ 0.000000e+00, [[LOOP_1]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[LOOP_1_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_1]] ]
+; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ @glob.1, [[LOOP_1_PH]] ], [ [[PTR_IV_1_NEXT:%.*]], [[LOOP_1]] ]
+; CHECK-NEXT: [[GEP_IV:%.*]] = getelementptr inbounds double, ptr @glob.2, i64 [[IV]]
+; CHECK-NEXT: [[L_1:%.*]] = load double, ptr [[GEP_IV]], align 8
+; CHECK-NEXT: [[GEP_IV_1:%.*]] = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 [[IV]]
+; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_IV_1]], align 8
+; CHECK-NEXT: store double 0.000000e+00, ptr [[DST_1]], align 8
+; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_1]], i64 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[LOOP_2_PH_LOOPEXIT3:%.*]], label [[LOOP_1]]
+; CHECK: loop.2.ph.loopexit:
+; CHECK-NEXT: [[LCSSA_PTR_IV_1_PH:%.*]] = phi ptr [ [[PTR_IV_1_LVER_ORIG]], [[LOOP_1_LVER_ORIG]] ]
+; CHECK-NEXT: br label [[LOOP_2_PH:%.*]]
+; CHECK: loop.2.ph.loopexit3:
+; CHECK-NEXT: [[LCSSA_PTR_IV_1_PH4:%.*]] = phi ptr [ [[PTR_IV_1]], [[LOOP_1]] ]
+; CHECK-NEXT: br label [[LOOP_2_PH]]
+; CHECK: loop.2.ph:
+; CHECK-NEXT: [[LCSSA_PTR_IV_1:%.*]] = phi ptr [ [[LCSSA_PTR_IV_1_PH]], [[LOOP_2_PH_LOOPEXIT]] ], [ [[LCSSA_PTR_IV_1_PH4]], [[LOOP_2_PH_LOOPEXIT3]] ]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], 8
+; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP2]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i32 0
+; CHECK-NEXT: store <4 x double> zeroinitializer, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_2_PH]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[LCSSA_PTR_IV_1]], [[LOOP_2_PH]] ]
+; CHECK-NEXT: br label [[LOOP_2:%.*]]
+; CHECK: loop.2:
+; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ]
+; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], [[LOOP_2]] ]
+; CHECK-NEXT: store double 0.000000e+00, ptr [[PTR_IV_2]], align 8
+; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_2]], i64 1
+; CHECK-NEXT: [[IV_2_NEXT]] = add nuw nsw i64 [[IV_2]], 1
+; CHECK-NEXT: [[EXITCOND_1_NOT:%.*]] = icmp eq i64 [[IV_2_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_1_NOT]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_2]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop.1
+
+loop.1:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.1 ]
+ %ptr.iv.1 = phi ptr [ @glob.1, %entry ], [ %ptr.iv.1.next, %loop.1 ]
+ %gep.iv = getelementptr inbounds double, ptr @glob.2, i64 %iv
+ %l.1 = load double, ptr %gep.iv, align 8
+ %gep.iv.1 = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 %iv
+ store double 0.000000e+00, ptr %gep.iv.1, align 8
+ store double 0.000000e+00, ptr %dst.1, align 8
+ %ptr.iv.1.next = getelementptr inbounds double, ptr %ptr.iv.1, i64 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv, %N
+ br i1 %exitcond.not, label %loop.2.ph, label %loop.1
+
+loop.2.ph:
+ %lcssa.ptr.iv.1 = phi ptr [ %ptr.iv.1, %loop.1 ]
+ br label %loop.2
+
+loop.2:
+ %iv.2 = phi i64 [ 0, %loop.2.ph ] , [ %iv.2.next, %loop.2 ]
+ %ptr.iv.2 = phi ptr [ %lcssa.ptr.iv.1, %loop.2.ph ], [ %ptr.iv.2.next, %loop.2 ]
+ store double 0.000000e+00, ptr %ptr.iv.2, align 8
+ %ptr.iv.2.next = getelementptr inbounds double, ptr %ptr.iv.2, i64 1
+ %iv.2.next = add nuw nsw i64 %iv.2, 1
+ %exitcond.1.not = icmp eq i64 %iv.2.next, %N
+ br i1 %exitcond.1.not, label %exit, label %loop.2
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list