[llvm] [WIP][Support] Experimental: Always scale InstructionCost::Value (PR #178962)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 5 21:09:07 PST 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/178962
>From 8768038e113ccfd79ad9621b4001b7d300ec2f2b Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 30 Jan 2026 12:26:39 -0800
Subject: [PATCH 1/4] [Support] Always scale InstructionCost::Value
Allows for fractional Instruction costs up to a granularity with little overhead.
Likely that I need update some tests from other backends.
The only functional change because of this commit is that finer granularity is
now supported. In LoopVectorizer, there are some calculations that divide a cost
by some value. Before, a rounded answer was produced, but now the result is more
accurate since we can represent a fractional cost. For instance:
Before:
InstructionCost(3) / 6 = 0
After (with ScalingFactor 120):
InstructionCost(3) / 6 = 60 / 120
Also, there is a decrease in the maximum value of InstructionCost, as the largest
value is now `std::numeric_limits<CostType>::max() / ScalingFactor`.
---
llvm/include/llvm/Support/InstructionCost.h | 30 ++++++-
llvm/lib/Support/InstructionCost.cpp | 2 +-
.../interchange-refcost-overflow.ll | 6 +-
.../LoopVectorize/AArch64/call-costs.ll | 84 +++++++++++++++----
.../LoopVectorize/RISCV/predicated-costs.ll | 32 ++++---
5 files changed, 120 insertions(+), 34 deletions(-)
diff --git a/llvm/include/llvm/Support/InstructionCost.h b/llvm/include/llvm/Support/InstructionCost.h
index 507c16666b958..a8237694050f9 100644
--- a/llvm/include/llvm/Support/InstructionCost.h
+++ b/llvm/include/llvm/Support/InstructionCost.h
@@ -59,6 +59,10 @@ class InstructionCost {
State = Invalid;
}
+ // 120 chosen since least common factor of 2, 3, 4, 5, 6, 8
+ // which are realistic issue widths
+ static constexpr CostType ScalingFactor = 120;
+
static constexpr CostType MaxValue = std::numeric_limits<CostType>::max();
static constexpr CostType MinValue = std::numeric_limits<CostType>::min();
@@ -67,7 +71,16 @@ class InstructionCost {
InstructionCost() = default;
InstructionCost(CostState) = delete;
- InstructionCost(CostType Val) : Value(Val), State(Valid) {}
+ InstructionCost(CostType Val) : Value(), State(Valid) {
+ InstructionCost::CostType Result;
+ if (MulOverflow(Val, ScalingFactor, Result)) {
+ if (Val > 0)
+ Result = MaxValue;
+ else
+ Result = MinValue;
+ }
+ Value = Result;
+ }
static InstructionCost getMax() { return MaxValue; }
static InstructionCost getMin() { return MinValue; }
@@ -87,7 +100,7 @@ class InstructionCost {
/// and comparisons.
CostType getValue() const {
assert(isValid());
- return Value;
+ return Value / ScalingFactor;
}
/// For all of the arithmetic operators provided here any invalid state is
@@ -141,6 +154,8 @@ class InstructionCost {
Result = MaxValue;
else
Result = MinValue;
+ } else {
+ Result /= ScalingFactor;
}
Value = Result;
@@ -155,7 +170,16 @@ class InstructionCost {
InstructionCost &operator/=(const InstructionCost &RHS) {
propagateState(RHS);
- Value /= RHS.Value;
+ // Saturating multiply.
+ InstructionCost::CostType Result;
+ if (MulOverflow(Value, ScalingFactor, Result)) {
+ if (Value > 0)
+ Result = MaxValue;
+ else
+ Result = MinValue;
+ }
+ Result /= RHS.Value;
+ Value = Result;
return *this;
}
diff --git a/llvm/lib/Support/InstructionCost.cpp b/llvm/lib/Support/InstructionCost.cpp
index c485ce9107af9..6b4eb6d2f1ed6 100644
--- a/llvm/lib/Support/InstructionCost.cpp
+++ b/llvm/lib/Support/InstructionCost.cpp
@@ -18,7 +18,7 @@ using namespace llvm;
void InstructionCost::print(raw_ostream &OS) const {
if (isValid())
- OS << Value;
+ OS << (Value / ScalingFactor);
else
OS << "Invalid";
}
diff --git a/llvm/test/Analysis/LoopCacheAnalysis/interchange-refcost-overflow.ll b/llvm/test/Analysis/LoopCacheAnalysis/interchange-refcost-overflow.ll
index 52a530b2feebb..88f265872902e 100644
--- a/llvm/test/Analysis/LoopCacheAnalysis/interchange-refcost-overflow.ll
+++ b/llvm/test/Analysis/LoopCacheAnalysis/interchange-refcost-overflow.ll
@@ -10,9 +10,9 @@
; A[c][d][e] = 0;
; }
-; CHECK: Loop 'outer.loop' has cost = 9223372036854775807
-; CHECK: Loop 'middle.loop' has cost = 9223372036854775807
-; CHECK: Loop 'inner.loop' has cost = 9223372036854775807
+; CHECK: Loop 'outer.loop' has cost = 76861433640456465
+; CHECK: Loop 'middle.loop' has cost = 76861433640456465
+; CHECK: Loop 'inner.loop' has cost = 76861433640456465
@A = local_unnamed_addr global [11 x [11 x [11 x i32]]] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index 95b4dcb23dd47..abb215ae595d7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -104,26 +104,81 @@ exit:
define void @call_scalarized(ptr noalias %src, ptr noalias %dst) {
; CHECK-LABEL: define void @call_scalarized(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = sub i64 100, [[INDEX]]
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], -1
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV_NEXT]]
-; CHECK-NEXT: [[L:%.*]] = load double, ptr [[GEP_SRC]], align 8
-; CHECK-NEXT: [[CMP295:%.*]] = fcmp une double [[L]], 4.000000e+00
-; CHECK-NEXT: [[CMP299:%.*]] = fcmp ugt double [[L]], 0.000000e+00
-; CHECK-NEXT: [[OR_COND:%.*]] = or i1 [[CMP295]], [[CMP299]]
-; CHECK-NEXT: br i1 [[OR_COND]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
-; CHECK: [[THEN]]:
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[GEP_SRC]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[TMP2]], i64 -1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[GEP_SRC]], i64 -2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[TMP4]], i64 -1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <2 x double> [[WIDE_LOAD]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <2 x double> [[WIDE_LOAD1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp une <2 x double> [[REVERSE]], splat (double 4.000000e+00)
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp une <2 x double> [[REVERSE2]], splat (double 4.000000e+00)
+; CHECK-NEXT: [[TMP8:%.*]] = fcmp ugt <2 x double> [[REVERSE]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp ugt <2 x double> [[REVERSE2]], zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = or <2 x i1> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = or <2 x i1> [[TMP7]], [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[TMP10]], splat (i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = xor <2 x i1> [[TMP11]], splat (i1 true)
+; CHECK-NEXT: [[OR_COND:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[LOOP_LATCH:.*]], label %[[THEN:.*]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[IV]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP15]], -1
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NEXT: [[L:%.*]] = extractelement <2 x double> [[REVERSE]], i32 0
; CHECK-NEXT: [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[L]])
-; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV_NEXT]]
; CHECK-NEXT: store double [[SQRT]], ptr [[GEP_DST]], align 8
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: br label %[[THEN]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
+; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[IV]], -1
+; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[TMP21]], -1
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x double> [[REVERSE]], i32 1
+; CHECK-NEXT: [[TMP25:%.*]] = call double @llvm.sqrt.f64(double [[TMP24]])
+; CHECK-NEXT: store double [[TMP25]], ptr [[TMP23]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x i1> [[TMP13]], i32 0
+; CHECK-NEXT: br i1 [[TMP26]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[IV]], -2
+; CHECK-NEXT: [[TMP28:%.*]] = add i64 [[TMP27]], -1
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP28]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x double> [[REVERSE2]], i32 0
+; CHECK-NEXT: [[TMP31:%.*]] = call double @llvm.sqrt.f64(double [[TMP30]])
+; CHECK-NEXT: store double [[TMP31]], ptr [[TMP29]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <2 x i1> [[TMP13]], i32 1
+; CHECK-NEXT: br i1 [[TMP32]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[TMP33:%.*]] = add i64 [[IV]], -3
+; CHECK-NEXT: [[TMP34:%.*]] = add i64 [[TMP33]], -1
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <2 x double> [[REVERSE2]], i32 1
+; CHECK-NEXT: [[TMP37:%.*]] = call double @llvm.sqrt.f64(double [[TMP36]])
+; CHECK-NEXT: store double [[TMP37]], ptr [[TMP35]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -211,4 +266,5 @@ declare i64 @llvm.fshl.i64(i64, i64, i64)
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
index e0ab30b0ae5cc..2316a478becc5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
@@ -8,23 +8,29 @@
define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-LABEL: define void @nested(
; CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], i1 [[C0:%.*]], i1 [[C1:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN_0:.*]], label %[[LATCH]]
-; CHECK: [[THEN_0]]:
-; CHECK-NEXT: br i1 [[C1]], label %[[THEN_1:.*]], label %[[LATCH]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: br label %[[THEN_1:.*]]
; CHECK: [[THEN_1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[THEN_1]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[LOOP]] ], [ [[AVL_NEXT:%.*]], %[[THEN_1]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P0]], i32 [[IV1]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP2]], align 4
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[P1]], i32 [[X]]
-; CHECK-NEXT: store i32 0, ptr [[GEP1]], align 4
-; CHECK-NEXT: br label %[[LATCH]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[GEP2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P1]], <vscale x 4 x i32> [[VP_OP_LOAD]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP3]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[IV1]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[LATCH:.*]], label %[[THEN_1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV1]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], 1024
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -80,7 +86,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
>From 74f68f0a3ba1f054b59f76a9c69bb3b57d23b90a Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 4 Feb 2026 16:28:46 -0800
Subject: [PATCH 2/4] [X86] Update X86 tests
---
.../LoopVectorize/X86/cost-model-i386.ll | 49 ++++-
.../LoopVectorize/X86/masked_load_store.ll | 68 ++++---
.../LoopVectorize/X86/predicate-switch.ll | 185 ++++++++++++++++--
.../X86/replicate-uniform-call.ll | 44 ++++-
.../X86/pr48844-br-to-switch-vectorization.ll | 82 +++++++-
5 files changed, 377 insertions(+), 51 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model-i386.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model-i386.ll
index 14f20464093cf..e53fdbc12d919 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model-i386.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model-i386.ll
@@ -6,26 +6,63 @@ target triple = "i386-unknow-linux"
define void @icmp_predicate_and_branch_cost(i32 %size, ptr %dst, i64 %conv5.i) #0 {
; CHECK-LABEL: @icmp_predicate_and_branch_cost(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[SIZE:%.*]], 7
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], 16
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[N_VEC]], 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[CONV5_I:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i32> poison, i32 [[SIZE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT1]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_HEADER]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i32> [ <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 64, i32 72, i32 80, i32 88, i32 96, i32 104, i32 112, i32 120>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_HEADER]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i32> [[VEC_IND]] to <16 x i64>
+; CHECK-NEXT: [[TMP5:%.*]] = add <16 x i64> [[TMP4]], splat (i64 8)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt <16 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp uge <16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP8:%.*]] = or <16 x i32> [[BROADCAST_SPLAT2]], [[VEC_IND]]
+; CHECK-NEXT: [[TMP9:%.*]] = trunc <16 x i32> [[TMP8]] to <16 x i8>
+; CHECK-NEXT: [[TMP10:%.*]] = select <16 x i1> [[TMP6]], <16 x i1> [[TMP7]], <16 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <16 x i1> [[TMP10]], <16 x i8> zeroinitializer, <16 x i8> [[TMP9]]
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <16 x i1> [[TMP6]], <16 x i8> [[PREDPHI]], <16 x i8> splat (i8 1)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <16 x i8> [[PREDPHI3]], i32 15
+; CHECK-NEXT: store i8 [[TMP11]], ptr [[DST:%.*]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <16 x i32> [[VEC_IND]], splat (i32 128)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[LOOP_HEADER1:%.*]]
; CHECK: loop.header:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[EXT_IV:%.*]] = zext i32 [[IV]] to i64
; CHECK-NEXT: [[ADD_IV:%.*]] = add i64 [[EXT_IV]], 8
-; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i64 [[ADD_IV]], [[CONV5_I:%.*]]
+; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i64 [[ADD_IV]], [[CONV5_I]]
; CHECK-NEXT: br i1 [[C_1]], label [[THEN_1:%.*]], label [[LOOP_LATCH]]
; CHECK: then.1:
-; CHECK-NEXT: [[C_2:%.*]] = icmp ult i32 [[IV]], [[SIZE:%.*]]
+; CHECK-NEXT: [[C_2:%.*]] = icmp ult i32 [[IV]], [[SIZE]]
; CHECK-NEXT: br i1 [[C_2]], label [[THEN_2:%.*]], label [[LOOP_LATCH]]
; CHECK: then.2:
; CHECK-NEXT: [[OR:%.*]] = or i32 [[SIZE]], [[IV]]
; CHECK-NEXT: [[TRUNC_OR:%.*]] = trunc i32 [[OR]] to i8
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.latch:
-; CHECK-NEXT: [[SINK:%.*]] = phi i8 [ [[TRUNC_OR]], [[THEN_2]] ], [ 1, [[LOOP_HEADER]] ], [ 0, [[THEN_1]] ]
-; CHECK-NEXT: store i8 [[SINK]], ptr [[DST:%.*]], align 1
+; CHECK-NEXT: [[SINK:%.*]] = phi i8 [ [[TRUNC_OR]], [[THEN_2]] ], [ 1, [[LOOP_HEADER1]] ], [ 0, [[THEN_1]] ]
+; CHECK-NEXT: store i8 [[SINK]], ptr [[DST]], align 1
; CHECK-NEXT: [[IV_NEXT]] = trunc i64 [[ADD_IV]] to i32
; CHECK-NEXT: [[C_3:%.*]] = icmp ugt i32 [[SIZE]], [[IV]]
-; CHECK-NEXT: br i1 [[C_3]], label [[LOOP_HEADER]], label [[EXIT:%.*]]
+; CHECK-NEXT: br i1 [[C_3]], label [[LOOP_HEADER1]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 1808e80a97060..6dc5813720a89 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -1073,27 +1073,49 @@ for.end: ; preds = %for.inc
define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr nocapture readonly %trigger) local_unnamed_addr #0 {
; AVX1-LABEL: define void @foo6(
; AVX1-SAME: ptr readonly captures(none) [[IN:%.*]], ptr captures(none) [[OUT:%.*]], i32 [[SIZE:%.*]], ptr readonly captures(none) [[TRIGGER:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; AVX1-NEXT: [[ENTRY:.*]]:
+; AVX1-NEXT: [[ENTRY:.*:]]
; AVX1-NEXT: br label %[[FOR_BODY:.*]]
; AVX1: [[FOR_BODY]]:
-; AVX1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 4095, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; AVX1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDVARS_IV]]
-; AVX1-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; AVX1-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
-; AVX1-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
-; AVX1: [[IF_THEN]]:
-; AVX1-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[IN]], i64 [[INDVARS_IV]]
-; AVX1-NEXT: [[TMP1:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
-; AVX1-NEXT: [[ADD:%.*]] = fadd double [[TMP1]], 5.000000e-01
-; AVX1-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[INDVARS_IV]]
-; AVX1-NEXT: store double [[ADD]], ptr [[ARRAYIDX5]], align 8
-; AVX1-NEXT: br label %[[FOR_INC]]
+; AVX1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 32768
+; AVX1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TRIGGER]], i64 16384
+; AVX1-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[IN]], i64 32768
+; AVX1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
+; AVX1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[TRIGGER]], [[SCEVGEP]]
+; AVX1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX1-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP2]]
+; AVX1-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP]]
+; AVX1-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; AVX1-NEXT: [[CMP1:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; AVX1-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[FOR_INC:.*]]
; AVX1: [[FOR_INC]]:
-; AVX1-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; AVX1-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
-; AVX1-NEXT: br i1 [[CMP]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; AVX1-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX1: [[VECTOR_BODY]]:
+; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_INC]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX1-NEXT: [[OFFSET_IDX:%.*]] = sub i64 4095, [[INDEX]]
+; AVX1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[OFFSET_IDX]]
+; AVX1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0
+; AVX1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 -3
+; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META18:![0-9]+]]
+; AVX1-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i32> [[REVERSE]], zeroinitializer
+; AVX1-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[IN]], i64 [[OFFSET_IDX]]
+; AVX1-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[TMP4]], i64 0
+; AVX1-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 -3
+; AVX1-NEXT: [[REVERSE6:%.*]] = shufflevector <4 x i1> [[TMP3]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP6]], <4 x i1> [[REVERSE6]], <4 x double> poison), !alias.scope [[META21:![0-9]+]]
+; AVX1-NEXT: [[REVERSE7:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT: [[TMP7:%.*]] = fadd <4 x double> [[REVERSE7]], splat (double 5.000000e-01)
+; AVX1-NEXT: [[TMP8:%.*]] = getelementptr double, ptr [[OUT]], i64 [[OFFSET_IDX]]
+; AVX1-NEXT: [[TMP9:%.*]] = getelementptr double, ptr [[TMP8]], i64 0
+; AVX1-NEXT: [[TMP10:%.*]] = getelementptr double, ptr [[TMP9]], i64 -3
+; AVX1-NEXT: [[REVERSE8:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[REVERSE8]], ptr align 8 [[TMP10]], <4 x i1> [[REVERSE6]]), !alias.scope [[META23:![0-9]+]], !noalias [[META25:![0-9]+]]
+; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; AVX1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; AVX1-NEXT: br i1 [[TMP11]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; AVX1: [[FOR_END]]:
-; AVX1-NEXT: ret void
+; AVX1-NEXT: br [[FOR_END1:label %.*]]
+; AVX1: [[IF_THEN]]:
;
; AVX2-LABEL: define void @foo6(
; AVX2-SAME: ptr readonly captures(none) [[IN:%.*]], ptr captures(none) [[OUT:%.*]], i32 [[SIZE:%.*]], ptr readonly captures(none) [[TRIGGER:%.*]]) local_unnamed_addr #[[ATTR0]] {
@@ -1373,13 +1395,13 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP39]], <4 x i1> [[TMP34]])
; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; AVX1-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX1-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; AVX1-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; AVX1: [[MIDDLE_BLOCK]]:
; AVX1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; AVX1-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; AVX1: [[VEC_EPILOG_ITER_CHECK]]:
; AVX1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
-; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF19:![0-9]+]]
+; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF29:![0-9]+]]
; AVX1: [[VEC_EPILOG_PH]]:
; AVX1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; AVX1-NEXT: [[N_MOD_VF8:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
@@ -1399,7 +1421,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP52]], <4 x i1> [[TMP51]])
; AVX1-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4
; AVX1-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]]
-; AVX1-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; AVX1-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; AVX1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; AVX1-NEXT: [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
; AVX1-NEXT: br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
@@ -1694,13 +1716,13 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP39]], <4 x i1> [[TMP34]])
; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; AVX1-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX1-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; AVX1-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; AVX1: [[MIDDLE_BLOCK]]:
; AVX1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; AVX1-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; AVX1: [[VEC_EPILOG_ITER_CHECK]]:
; AVX1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
-; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF19]]
+; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF29]]
; AVX1: [[VEC_EPILOG_PH]]:
; AVX1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; AVX1-NEXT: [[N_MOD_VF8:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
@@ -1720,7 +1742,7 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP52]], <4 x i1> [[TMP51]])
; AVX1-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4
; AVX1-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]]
-; AVX1-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; AVX1-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; AVX1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; AVX1-NEXT: [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
; AVX1-NEXT: br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
index 5a396f88b1a64..75a0623a2e0d3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
@@ -288,9 +288,46 @@ define void @switch_all_dests_distinct(ptr %start, ptr %end) {
; COST-LABEL: define void @switch_all_dests_distinct(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_HEADER]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer
+; COST-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
+; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 1), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP8]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP6]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP11]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; COST-NEXT: br label %[[LOOP_HEADER1:.*]]
+; COST: [[LOOP_HEADER1]]:
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -312,7 +349,7 @@ define void @switch_all_dests_distinct(ptr %start, ptr %end) {
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP7:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -469,7 +506,7 @@ define void @switch_all_dests_distinct_variant_using_branches(ptr %start, ptr %e
; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COST-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COST: [[MIDDLE_BLOCK]]:
; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -502,7 +539,7 @@ define void @switch_all_dests_distinct_variant_using_branches(ptr %start, ptr %e
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -639,9 +676,49 @@ define void @switch_multiple_common_dests(ptr %start, ptr %end) {
; COST-LABEL: define void @switch_multiple_common_dests(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_HEADER]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer
+; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP9:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 14)
+; COST-NEXT: [[TMP10:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 15)
+; COST-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
+; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP10]]
+; COST-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP11]], [[TMP13]]
+; COST-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[TMP14]], splat (i1 true)
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP13]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP11]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; COST-NEXT: br label %[[LOOP_HEADER1:.*]]
+; COST: [[LOOP_HEADER1]]:
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -662,7 +739,7 @@ define void @switch_multiple_common_dests(ptr %start, ptr %end) {
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP11:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -790,9 +867,43 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; COST-LABEL: define void @switch4_default_common_dest_with_case(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_HEADER]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP6]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP9]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; COST-NEXT: br label %[[LOOP_HEADER1:.*]]
+; COST: [[LOOP_HEADER1]]:
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -811,7 +922,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP13:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -957,7 +1068,7 @@ define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end,
; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; COST: [[MIDDLE_BLOCK]]:
; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -987,7 +1098,7 @@ define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end,
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -1116,9 +1227,51 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
; COST-LABEL: define void @br_under_switch_default_common_dest_with_case(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; COST-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; COST-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_HEADER]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; COST-NEXT: [[TMP10:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
+; COST-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
+; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP7]]
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP13]])
+; COST-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]])
+; COST-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; COST-NEXT: br label %[[LOOP_HEADER1:.*]]
+; COST: [[LOOP_HEADER1]]:
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -1140,7 +1293,7 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP17:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -1433,6 +1586,14 @@ exit:
; COST: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
; COST: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
; COST: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; COST: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; COST: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; COST: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; COST: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+; COST: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; COST: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
+; COST: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
+; COST: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
;.
; FORCED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; FORCED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
index 611a097024134..53c6dccddc710 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
@@ -8,26 +8,49 @@ target triple = "x86_64-unknown-linux-gnu"
define void @smax_call_uniform(ptr %dst, i64 %x) {
; CHECK-LABEL: define void @smax_call_uniform(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[X:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[C:%.*]] = icmp ult i8 -68, -69
; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i64 [[X]], 0
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ELSE:.*]], label %[[LOOP_LATCH:.*]]
; CHECK: [[ELSE]]:
-; CHECK-NEXT: [[REM:%.*]] = urem i64 [[MUL]], [[X]]
-; CHECK-NEXT: [[SMAX:%.*]] = tail call i64 @llvm.smax.i64(i64 [[REM]], i64 0)
; CHECK-NEXT: br label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[PREDPHI7:%.*]] = phi i64 [ 1, %[[LOOP_HEADER]] ], [ [[SMAX]], %[[ELSE]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_UREM_IF1:.*]], label %[[PRED_UREM_CONTINUE2:.*]]
+; CHECK: [[PRED_UREM_IF1]]:
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2]]
+; CHECK: [[PRED_UREM_CONTINUE2]]:
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_UREM_IF3:.*]], label %[[PRED_UREM_CONTINUE4:.*]]
+; CHECK: [[PRED_UREM_IF3]]:
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4]]
+; CHECK: [[PRED_UREM_CONTINUE4]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6]]
+; CHECK: [[PRED_UREM_IF5]]:
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE6]]
+; CHECK: [[PRED_UREM_CONTINUE6]]:
+; CHECK-NEXT: [[TMP5:%.*]] = tail call i64 @llvm.smax.i64(i64 0, i64 0)
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C]], i64 1, i64 [[TMP5]]
; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[PREDPHI7]], 1
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP17]]
; CHECK-NEXT: store i64 0, ptr [[TMP19]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: store i64 0, ptr [[TMP19]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP20]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -58,3 +81,8 @@ exit:
}
declare i64 @llvm.smax.i64(i64, i64)
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
index dcfebe32302be..d8ce1d9be901e 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
@@ -12,8 +12,86 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
; AVX-NEXT: entry:
; AVX-NEXT: [[I11_NOT1:%.*]] = icmp eq ptr [[START:%.*]], [[END:%.*]]
; AVX-NEXT: br i1 [[I11_NOT1]], label [[EXIT:%.*]], label [[BB12:%.*]]
+; AVX: iter.check:
+; AVX-NEXT: [[END3:%.*]] = ptrtoint ptr [[END]] to i64
+; AVX-NEXT: [[START4:%.*]] = ptrtoint ptr [[START]] to i64
+; AVX-NEXT: [[TMP0:%.*]] = add i64 [[END3]], -4
+; AVX-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START4]]
+; AVX-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2
+; AVX-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; AVX-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 28
+; AVX-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[BB12_PREHEADER:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; AVX: vector.main.loop.iter.check:
+; AVX-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP1]], 124
+; AVX-NEXT: br i1 [[MIN_ITERS_CHECK5]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AVX: vector.ph:
+; AVX-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 24
+; AVX-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775776
+; AVX-NEXT: br label [[VECTOR_BODY:%.*]]
+; AVX: vector.body:
+; AVX-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT: [[TMP4:%.*]] = shl i64 [[INDEX]], 2
+; AVX-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; AVX-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 32
+; AVX-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 64
+; AVX-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 96
+; AVX-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[NEXT_GEP]], align 4
+; AVX-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
+; AVX-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4
+; AVX-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4
+; AVX-NEXT: [[TMP8:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD]], splat (i32 -12)
+; AVX-NEXT: [[TMP9:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD6]], splat (i32 -12)
+; AVX-NEXT: [[TMP10:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD7]], splat (i32 -12)
+; AVX-NEXT: [[TMP11:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD8]], splat (i32 -12)
+; AVX-NEXT: [[TMP12:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD]], splat (i32 13)
+; AVX-NEXT: [[TMP13:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD6]], splat (i32 13)
+; AVX-NEXT: [[TMP14:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD7]], splat (i32 13)
+; AVX-NEXT: [[TMP15:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD8]], splat (i32 13)
+; AVX-NEXT: [[TMP16:%.*]] = or <8 x i1> [[TMP8]], [[TMP12]]
+; AVX-NEXT: [[TMP17:%.*]] = or <8 x i1> [[TMP9]], [[TMP13]]
+; AVX-NEXT: [[TMP18:%.*]] = or <8 x i1> [[TMP10]], [[TMP14]]
+; AVX-NEXT: [[TMP19:%.*]] = or <8 x i1> [[TMP11]], [[TMP15]]
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[NEXT_GEP]], <8 x i1> [[TMP16]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP5]], <8 x i1> [[TMP17]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP6]], <8 x i1> [[TMP18]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP7]], <8 x i1> [[TMP19]])
+; AVX-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; AVX-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX: middle.block:
+; AVX-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; AVX-NEXT: br i1 [[CMP_N]], label [[EXIT]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; AVX: vec.epilog.iter.check:
+; AVX-NEXT: [[TMP21:%.*]] = shl i64 [[N_VEC]], 2
+; AVX-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP21]]
+; AVX-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; AVX-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[BB12_PREHEADER]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; AVX: vec.epilog.ph:
+; AVX-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX-NEXT: [[N_VEC10:%.*]] = and i64 [[TMP3]], 9223372036854775800
+; AVX-NEXT: [[TMP22:%.*]] = shl i64 [[N_VEC10]], 2
+; AVX-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP22]]
+; AVX-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; AVX: vec.epilog.vector.body:
+; AVX-NEXT: [[INDEX11:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX11]], 2
+; AVX-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; AVX-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i32>, ptr [[NEXT_GEP12]], align 4
+; AVX-NEXT: [[TMP24:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 -12)
+; AVX-NEXT: [[TMP25:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 13)
+; AVX-NEXT: [[TMP26:%.*]] = or <8 x i1> [[TMP24]], [[TMP25]]
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[NEXT_GEP12]], <8 x i1> [[TMP26]])
+; AVX-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX11]], 8
+; AVX-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC10]]
+; AVX-NEXT: br i1 [[TMP27]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AVX: vec.epilog.middle.block:
+; AVX-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]]
+; AVX-NEXT: br i1 [[CMP_N15]], label [[EXIT]], label [[BB12_PREHEADER]]
+; AVX: bb12.preheader:
+; AVX-NEXT: [[PTR2_PH:%.*]] = phi ptr [ [[START]], [[BB12]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP23]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; AVX-NEXT: br label [[BB13:%.*]]
; AVX: bb12:
-; AVX-NEXT: [[PTR2:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LATCH:%.*]] ], [ [[START]], [[ENTRY:%.*]] ]
+; AVX-NEXT: [[PTR2:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LATCH:%.*]] ], [ [[PTR2_PH]], [[BB12_PREHEADER]] ]
; AVX-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR2]], align 4
; AVX-NEXT: switch i32 [[VAL]], label [[LATCH]] [
; AVX-NEXT: i32 -12, label [[STORE:%.*]]
@@ -25,7 +103,7 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
; AVX: latch:
; AVX-NEXT: [[PTR_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR2]], i64 4
; AVX-NEXT: [[I11_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[END]]
-; AVX-NEXT: br i1 [[I11_NOT]], label [[EXIT]], label [[BB12]]
+; AVX-NEXT: br i1 [[I11_NOT]], label [[EXIT]], label [[BB13]], !llvm.loop [[LOOP5:![0-9]+]]
; AVX: exit:
; AVX-NEXT: ret void
;
>From dbc63b8065a5797a527e4c0227e70d1616e032dc Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 4 Feb 2026 16:29:10 -0800
Subject: [PATCH 3/4] [Support] Optimzie InstructionCost for compile time
---
llvm/include/llvm/Support/InstructionCost.h | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/Support/InstructionCost.h b/llvm/include/llvm/Support/InstructionCost.h
index a8237694050f9..f6cc41ebb61ef 100644
--- a/llvm/include/llvm/Support/InstructionCost.h
+++ b/llvm/include/llvm/Support/InstructionCost.h
@@ -65,6 +65,8 @@ class InstructionCost {
static constexpr CostType MaxValue = std::numeric_limits<CostType>::max();
static constexpr CostType MinValue = std::numeric_limits<CostType>::min();
+ static constexpr CostType MaxInputValue = MaxValue / ScalingFactor;
+ static constexpr CostType MinInputValue = MinValue / ScalingFactor;
public:
// A default constructed InstructionCost is a valid zero cost
@@ -72,14 +74,13 @@ class InstructionCost {
InstructionCost(CostState) = delete;
InstructionCost(CostType Val) : Value(), State(Valid) {
- InstructionCost::CostType Result;
- if (MulOverflow(Val, ScalingFactor, Result)) {
- if (Val > 0)
- Result = MaxValue;
- else
- Result = MinValue;
- }
- Value = Result;
+ if (Val > MaxInputValue) [[unlikely]]
+ Val = MaxValue;
+ else if (Val < MinInputValue) [[unlikely]]
+ Val = MinValue;
+ else [[likely]]
+ Val *= ScalingFactor;
+ Value = Val;
}
static InstructionCost getMax() { return MaxValue; }
@@ -184,8 +185,7 @@ class InstructionCost {
}
InstructionCost &operator/=(const CostType RHS) {
- InstructionCost RHS2(RHS);
- *this /= RHS2;
+ Value /= RHS;
return *this;
}
>From f7ceb864113ea51631f8603be3a5087ca4f93988 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 5 Feb 2026 17:26:11 -0800
Subject: [PATCH 4/4] [utils] Adjust InstructionCost::ScalingFactor to 4 to
optimize for compile time
Multiplies and divides can be just shifted now.
---
llvm/include/llvm/Support/InstructionCost.h | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/Support/InstructionCost.h b/llvm/include/llvm/Support/InstructionCost.h
index f6cc41ebb61ef..0b26e22340e0f 100644
--- a/llvm/include/llvm/Support/InstructionCost.h
+++ b/llvm/include/llvm/Support/InstructionCost.h
@@ -59,9 +59,8 @@ class InstructionCost {
State = Invalid;
}
- // 120 chosen since least common factor of 2, 3, 4, 5, 6, 8
- // which are realistic issue widths
- static constexpr CostType ScalingFactor = 120;
+ // Matches GCC, can use shift rather than multiply/divide to scale
+ static constexpr CostType ScalingFactor = 4;
static constexpr CostType MaxValue = std::numeric_limits<CostType>::max();
static constexpr CostType MinValue = std::numeric_limits<CostType>::min();
More information about the llvm-commits
mailing list