[llvm] [VPlan] Extract IV exit value from vector loop if predicates are needed. (PR #200020)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 12:51:37 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/200020
>From 77bed0e70b2b048978f69f34794fe4b037df1e46 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 27 May 2026 18:51:48 +0100
Subject: [PATCH] [VPlan] Extract IV exit value from vector loop if predicates
are needed.
When an IV needs predicates (e.g. to make sure it does not wrap in a
narrower range), the precomputed end value would use the inductions
narrowed step, which can lead to incorrect results for the exiting value
outside the loop. For example consider a loop where IV has a predicate
to not wrap in i16 with the top 16 bits stripped (something like
https://llvm.godbolt.org/z/nzfoWbcEG).
Building on top of tracking the predicates per-IV
(https://github.com/llvm/llvm-project/pull/192876), we can simply
extract the exit values from the vector values in the loop.
---
.../Vectorize/VPlanConstruction.cpp | 22 ---
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +
llvm/test/Transforms/LoopVectorize/pr33706.ll | 85 +++++++++--
.../LoopVectorize/predicated-inductions.ll | 135 ++++++++++++++----
4 files changed, 193 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 79713a2ee4a98..e0b92aa23f147 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -996,28 +996,6 @@ bool VPlanTransforms::finalizeSCEVPredicates(VPlan &Plan,
PSE.addPredicate(*P);
}
- // Bail out if exit phis use predicated IVs via ExitingIVValue, as the
- // predicated SCEV may not hold outside the loop (PR33706). Check each IV's
- // predicates directly, regardless of whether PSE was already non-trivial
- // from other sources (e.g., LAI predicates).
- // TODO: Overly conservative; the pre-computed exit values are not correct
- // outside the loop, but the exit values could be extracted from the vector
- // loop.
- if (!PredicatedIVs.empty())
- for (auto *EB : Plan.getExitBlocks())
- for (VPRecipeBase &R : EB->phis())
- for (VPValue *Op : R.operands()) {
- VPValue *Inner;
- if (!match(Op, m_ExitingIVValue(m_VPValue(Inner))))
- continue;
- auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Inner);
- if (!WideIV || !PredicatedIVs.contains(WideIV))
- continue;
- LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Predicated IV has "
- "outside-loop use via ExitingIVValue\n");
- return false;
- }
-
unsigned TotalComplexity = PSE.getPredicate().getComplexity();
if (TotalComplexity && OptForSize) {
LLVM_DEBUG(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1427a1dc2f729..fd7092a45b367 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1109,6 +1109,12 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
if (!WideIV)
return nullptr;
+ // Skip pre-computing the final value for IVs that require SCEV predicates;
+ // their SCEV-derived end value may rely on predicates that only hold inside
+ // the loop.
+ if (Incoming == WideIV && !WideIV->getNoWrapPredicates().empty())
+ return nullptr;
+
VPValue *EndValue = EndValues.lookup(WideIV);
assert(EndValue && "Must have computed the end value up front");
diff --git a/llvm/test/Transforms/LoopVectorize/pr33706.ll b/llvm/test/Transforms/LoopVectorize/pr33706.ll
index 59fd1eec0a746..75fd9a8913d6a 100644
--- a/llvm/test/Transforms/LoopVectorize/pr33706.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr33706.ll
@@ -20,12 +20,81 @@ define void @PR33706(ptr nocapture readonly %arg, ptr nocapture %arg1, i32 %arg2
; CHECK-NEXT: br label [[BB27:%.*]]
; CHECK: bb9:
; CHECK-NEXT: [[VAR_TMP10:%.*]] = udiv i32 65536, [[ARG2]]
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[ARG2]] to i16
+; CHECK-NEXT: [[TMP1:%.*]] = sext i16 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[VAR_TMP10]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK: vector.scevcheck:
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[VAR_TMP10]], -1
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[ARG2]] to i16
+; CHECK-NEXT: [[TMP7:%.*]] = sub i16 0, [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP]] to i16
+; CHECK-NEXT: [[TMP9:%.*]] = icmp slt i16 [[TMP6]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i16 [[TMP7]], i16 [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i32 [[TMP5]] to i16
+; CHECK-NEXT: [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[TMP10]], i16 [[TMP11]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = add i16 [[TMP8]], [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP13:%.*]] = sub i16 [[TMP8]], [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i16 [[TMP12]], [[TMP8]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ugt i16 [[TMP13]], [[TMP8]]
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], i1 [[TMP15]], i1 [[TMP14]]
+; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i32 [[TMP5]], 65535
+; CHECK-NEXT: [[TMP19:%.*]] = icmp ne i16 [[TMP6]], 0
+; CHECK-NEXT: [[TMP20:%.*]] = and i1 [[TMP18]], [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = or i1 [[TMP17]], [[TMP20]]
+; CHECK-NEXT: br i1 [[TMP21]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP22:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP23:%.*]] = mul i32 [[TMP22]], [[TMP1]]
+; CHECK-NEXT: [[TMP24:%.*]] = add i32 [[VAR_TMP5]], [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[VAR_TMP4]], i64 [[TMP25]]
+; CHECK-NEXT: [[TMP27:%.*]] = sub i32 [[VAR_TMP10]], [[TMP22]]
+; CHECK-NEXT: [[TMP28:%.*]] = mul i32 [[TMP22]], [[ARG2]]
+; CHECK-NEXT: [[TMP29:%.*]] = add i32 [[TMP]], [[TMP28]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[VAR_TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT1]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP30:%.*]] = mul <2 x i32> <i32 0, i32 1>, [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[TMP30]]
+; CHECK-NEXT: [[TMP31:%.*]] = shl i32 [[TMP1]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i32> poison, i32 [[TMP31]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT3]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP32:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[VAR_TMP4]], i64 [[TMP32]]
+; CHECK-NEXT: [[TMP33:%.*]] = sitofp <2 x i32> [[VEC_IND]] to <2 x float>
+; CHECK-NEXT: store <2 x float> [[TMP33]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[BB22:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP24]], [[MIDDLE_BLOCK]] ], [ [[VAR_TMP5]], [[BB9]] ], [ [[VAR_TMP5]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP26]], [[MIDDLE_BLOCK]] ], [ [[VAR_TMP4]], [[BB9]] ], [ [[VAR_TMP4]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi i32 [ [[TMP27]], [[MIDDLE_BLOCK]] ], [ [[VAR_TMP10]], [[BB9]] ], [ [[VAR_TMP10]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ [[TMP]], [[BB9]] ], [ [[TMP]], [[VECTOR_SCEVCHECK]] ]
; CHECK-NEXT: br label [[BB11:%.*]]
; CHECK: bb11:
-; CHECK-NEXT: [[VAR_TMP12:%.*]] = phi i32 [ [[VAR_TMP20:%.*]], [[BB11]] ], [ [[VAR_TMP5]], [[BB9]] ]
-; CHECK-NEXT: [[VAR_TMP13:%.*]] = phi ptr [ [[VAR_TMP18:%.*]], [[BB11]] ], [ [[VAR_TMP4]], [[BB9]] ]
-; CHECK-NEXT: [[VAR_TMP14:%.*]] = phi i32 [ [[VAR_TMP16:%.*]], [[BB11]] ], [ [[VAR_TMP10]], [[BB9]] ]
-; CHECK-NEXT: [[VAR_TMP15:%.*]] = phi i32 [ [[VAR_TMP19:%.*]], [[BB11]] ], [ [[TMP]], [[BB9]] ]
+; CHECK-NEXT: [[VAR_TMP12:%.*]] = phi i32 [ [[VAR_TMP20:%.*]], [[BB11]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAR_TMP13:%.*]] = phi ptr [ [[VAR_TMP18:%.*]], [[BB11]] ], [ [[BC_RESUME_VAL5]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAR_TMP14:%.*]] = phi i32 [ [[VAR_TMP16:%.*]], [[BB11]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAR_TMP15:%.*]] = phi i32 [ [[VAR_TMP19:%.*]], [[BB11]] ], [ [[BC_RESUME_VAL7]], [[SCALAR_PH]] ]
; CHECK-NEXT: [[VAR_TMP16]] = add nsw i32 [[VAR_TMP14]], -1
; CHECK-NEXT: [[VAR_TMP17:%.*]] = sitofp i32 [[VAR_TMP12]] to float
; CHECK-NEXT: store float [[VAR_TMP17]], ptr [[VAR_TMP13]], align 4
@@ -33,11 +102,11 @@ define void @PR33706(ptr nocapture readonly %arg, ptr nocapture %arg1, i32 %arg2
; CHECK-NEXT: [[VAR_TMP19]] = add i32 [[VAR_TMP15]], [[ARG2]]
; CHECK-NEXT: [[VAR_TMP20]] = and i32 [[VAR_TMP19]], 65535
; CHECK-NEXT: [[VAR_TMP21:%.*]] = icmp eq i32 [[VAR_TMP16]], 0
-; CHECK-NEXT: br i1 [[VAR_TMP21]], label [[BB22:%.*]], label [[BB11]]
+; CHECK-NEXT: br i1 [[VAR_TMP21]], label [[BB22]], label [[BB11]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: bb22:
-; CHECK-NEXT: [[VAR_TMP23:%.*]] = phi ptr [ [[VAR_TMP18]], [[BB11]] ]
-; CHECK-NEXT: [[VAR_TMP24:%.*]] = phi i32 [ [[VAR_TMP19]], [[BB11]] ]
-; CHECK-NEXT: [[VAR_TMP25:%.*]] = phi i32 [ [[VAR_TMP20]], [[BB11]] ]
+; CHECK-NEXT: [[VAR_TMP23:%.*]] = phi ptr [ [[VAR_TMP18]], [[BB11]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAR_TMP24:%.*]] = phi i32 [ [[VAR_TMP19]], [[BB11]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAR_TMP25:%.*]] = phi i32 [ [[VAR_TMP20]], [[BB11]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[VAR_TMP26:%.*]] = ashr i32 [[VAR_TMP24]], 16
; CHECK-NEXT: store i32 [[VAR_TMP26]], ptr @global, align 4
; CHECK-NEXT: br label [[BB27]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
index 1496cc6618071..ea9391ad20d26 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 < %s | FileCheck --check-prefixes=COMMON,CHECK %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 < %s | FileCheck --check-prefixes=CHECK %s
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 \
-; RUN: -vectorize-scev-check-threshold=0 %s | FileCheck --check-prefixes=COMMON,THRESHOLD0 %s
+; RUN: -vectorize-scev-check-threshold=0 %s | FileCheck --check-prefixes=THRESHOLD0 %s
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 \
-; RUN: -vectorize-scev-check-threshold=1 %s | FileCheck --check-prefixes=COMMON,THRESHOLD1 %s
+; RUN: -vectorize-scev-check-threshold=1 %s | FileCheck --check-prefixes=THRESHOLD1 %s
define i64 @predicated_iv_with_liveout(ptr %dst, i64 %n) {
;
@@ -473,15 +473,17 @@ define i32 @predicated_iv_liveout_with_lai_preds(ptr %dst, ptr %src, i64 %stride
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 9, i32 18, i32 27>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 36)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[TMP7]], 9
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[VEC_IND]], i64 3
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -502,7 +504,7 @@ define i32 @predicated_iv_liveout_with_lai_preds(ptr %dst, ptr %src, i64 %stride
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_IV]], %[[LOOP]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_IV]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
; THRESHOLD0-LABEL: define i32 @predicated_iv_liveout_with_lai_preds(
@@ -572,26 +574,109 @@ exit:
define i32 @predicated_iv_inc_liveout_with_lai_preds(ptr %dst, ptr %src, i64 %stride, i64 %n) {
-; COMMON-LABEL: define i32 @predicated_iv_inc_liveout_with_lai_preds(
-; COMMON-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
-; COMMON-NEXT: [[ENTRY:.*]]:
-; COMMON-NEXT: br label %[[LOOP:.*]]
-; COMMON: [[LOOP]]:
-; COMMON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; COMMON-NEXT: [[PRED_IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PRED_NEXT:%.*]], %[[LOOP]] ]
-; COMMON-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; COMMON-NEXT: [[MASKED:%.*]] = and i32 [[PRED_IV]], 255
-; COMMON-NEXT: [[PRED_NEXT]] = add nuw nsw i32 [[MASKED]], 9
-; COMMON-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
-; COMMON-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_SRC]], align 4
-; COMMON-NEXT: [[STRIDE_OFF:%.*]] = mul nsw i64 [[IV]], [[STRIDE]]
-; COMMON-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[STRIDE_OFF]]
-; COMMON-NEXT: store i32 [[VAL]], ptr [[GEP_DST]], align 4
-; COMMON-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMMON-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
-; COMMON: [[EXIT]]:
-; COMMON-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_NEXT]], %[[LOOP]] ]
-; COMMON-NEXT: ret i32 [[RESULT]]
+; CHECK-LABEL: define i32 @predicated_iv_inc_liveout_with_lai_preds(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX3]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i8
+; CHECK-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 9, i8 [[TMP1]])
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[TMP0]], 255
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[MUL_OVERFLOW]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[IDENT_CHECK]], [[TMP3]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], 16
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX3]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX3]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], 9
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP9]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PRED_IV:%.*]] = phi i32 [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[PRED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MASKED:%.*]] = and i32 [[PRED_IV]], 255
+; CHECK-NEXT: [[PRED_NEXT]] = add nuw nsw i32 [[MASKED]], 9
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[STRIDE_OFF:%.*]] = mul nsw i64 [[IV]], [[STRIDE]]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[STRIDE_OFF]]
+; CHECK-NEXT: store i32 [[VAL]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+; THRESHOLD0-LABEL: define i32 @predicated_iv_inc_liveout_with_lai_preds(
+; THRESHOLD0-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; THRESHOLD0-NEXT: [[ENTRY:.*]]:
+; THRESHOLD0-NEXT: br label %[[LOOP:.*]]
+; THRESHOLD0: [[LOOP]]:
+; THRESHOLD0-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[PRED_IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PRED_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; THRESHOLD0-NEXT: [[MASKED:%.*]] = and i32 [[PRED_IV]], 255
+; THRESHOLD0-NEXT: [[PRED_NEXT]] = add nuw nsw i32 [[MASKED]], 9
+; THRESHOLD0-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; THRESHOLD0-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; THRESHOLD0-NEXT: [[STRIDE_OFF:%.*]] = mul nsw i64 [[IV]], [[STRIDE]]
+; THRESHOLD0-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[STRIDE_OFF]]
+; THRESHOLD0-NEXT: store i32 [[VAL]], ptr [[GEP_DST]], align 4
+; THRESHOLD0-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; THRESHOLD0-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; THRESHOLD0: [[EXIT]]:
+; THRESHOLD0-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_NEXT]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: ret i32 [[RESULT]]
+;
+; THRESHOLD1-LABEL: define i32 @predicated_iv_inc_liveout_with_lai_preds(
+; THRESHOLD1-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; THRESHOLD1-NEXT: [[ENTRY:.*]]:
+; THRESHOLD1-NEXT: br label %[[LOOP:.*]]
+; THRESHOLD1: [[LOOP]]:
+; THRESHOLD1-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD1-NEXT: [[PRED_IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PRED_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; THRESHOLD1-NEXT: [[MASKED:%.*]] = and i32 [[PRED_IV]], 255
+; THRESHOLD1-NEXT: [[PRED_NEXT]] = add nuw nsw i32 [[MASKED]], 9
+; THRESHOLD1-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; THRESHOLD1-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; THRESHOLD1-NEXT: [[STRIDE_OFF:%.*]] = mul nsw i64 [[IV]], [[STRIDE]]
+; THRESHOLD1-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[STRIDE_OFF]]
+; THRESHOLD1-NEXT: store i32 [[VAL]], ptr [[GEP_DST]], align 4
+; THRESHOLD1-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; THRESHOLD1-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; THRESHOLD1: [[EXIT]]:
+; THRESHOLD1-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_NEXT]], %[[LOOP]] ]
+; THRESHOLD1-NEXT: ret i32 [[RESULT]]
;
entry:
br label %loop
More information about the llvm-commits
mailing list