[llvm] 8a21afc - [LV] Add cost for VPInstruction::Not. (#198445)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 3 16:51:55 PDT 2026
Author: Elvis Wang
Date: 2026-06-03T23:51:50Z
New Revision: 8a21afca17117a380f2716ae81fb402d673f48fe
URL: https://github.com/llvm/llvm-project/commit/8a21afca17117a380f2716ae81fb402d673f48fe
DIFF: https://github.com/llvm/llvm-project/commit/8a21afca17117a380f2716ae81fb402d673f48fe.diff
LOG: [LV] Add cost for VPInstruction::Not. (#198445)
This patch add the cost for VPInstruction::Not which will lower to Xor.
Added:
llvm/test/Transforms/LoopVectorize/AArch64/arith-costs.ll
Modified:
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
Removed:
llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 8f9a390d1d893..1e32a948e1828 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1390,6 +1390,17 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
VecTy, Ctx.CostKind, 0);
}
+ case VPInstruction::Not: {
+ Type *ValTy = this->getScalarType();
+ // InstCombine will fold `xor` to the conditional branch.
+ if (auto *U = const_cast<VPUser *>(getSingleUser()))
+ if (match(U, m_BranchOnCond(m_VPValue())))
+ return 0;
+ if (!vputils::onlyFirstLaneUsed(this))
+ ValTy = toVectorTy(ValTy, VF);
+ return Ctx.TTI.getArithmeticInstrCost(Instruction::Xor, ValTy,
+ Ctx.CostKind);
+ }
case VPInstruction::BranchOnCount: {
// If TC <= VF then this is just a branch.
// FIXME: Removing the branch happens in simplifyBranchConditionForVFAndUF
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arith-costs.ll
similarity index 61%
rename from llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
rename to llvm/test/Transforms/LoopVectorize/AArch64/arith-costs.ll
index e42090d556f3c..9b02fd24c828d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/arith-costs.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "[Cc]ost.*(udiv|fneg|fmul)"
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "[Cc]ost.*(udiv|fneg|fmul|not)"
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize %s -S -o - 2>&1 | FileCheck %s
; REQUIRES: asserts
@@ -69,4 +69,44 @@ exit:
ret void
}
+define i32 @test_scalar_and_widen_not_cost(ptr readonly %a, i64 noundef %n) {
+; CHECK-LABEL: 'test_scalar_and_widen_not_cost'
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP11:%[0-9]+]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 2 for VF 32: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 32: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 4 for VF 64: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 64: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 8 for VF 128: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 128: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ]
+ %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ]
+ %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+ %load1 = load float, ptr %arrayidx, align 4
+ %cmp1 = fcmp olt float %load1, 0.000000e+00
+ %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+ %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %0 = select i1 %.any.0.off0, i32 2, i32 3
+ %1 = select i1 %all.off, i32 1, i32 %0
+ ret i32 %1
+}
+
attributes #0 = { "target-cpu"="neoverse-v2" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index 1ca68083d58ad..ef6fc9779c685 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -198,7 +198,7 @@ then:
loop.latch:
%tobool.not = icmp eq i64 %iv.next, 0
- br i1 %tobool.not, label %exit, label %loop.header
+ br i1 %tobool.not, label %exit, label %loop.header, !llvm.loop !0
exit:
ret void
@@ -277,7 +277,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -296,7 +296,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: store i32 [[COND]], ptr [[DST_GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -320,3 +320,6 @@ loop:
exit:
ret void
}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 2}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
index 5d66eeb25d7b5..922c51d2e31ef 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
@@ -38,7 +38,7 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
@@ -69,7 +69,7 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 4: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
@@ -121,7 +121,7 @@ define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi double [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 742c3850bf17c..341e19e36ca12 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1162,28 +1162,21 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 1
+; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 8)
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; DEFAULT-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP33:%.*]] = shl nuw i64 [[TMP29]], 1
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; DEFAULT-NEXT: [[TMP37:%.*]] = mul i64 [[TMP5]], 4
; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[C1]], [[A2]]
-; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP37]]
-; DEFAULT-NEXT: [[TMP41:%.*]] = mul i64 [[TMP5]], 4
+; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
; DEFAULT-NEXT: [[TMP7:%.*]] = sub i64 [[C1]], [[B3]]
-; DEFAULT-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP7]], [[TMP41]]
+; DEFAULT-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
; DEFAULT-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; DEFAULT-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[TMP15:%.*]] = shl nuw i64 [[TMP1]], 4
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP0]], [[TMP15]]
-; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK5]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; DEFAULT-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
; DEFAULT: [[VECTOR_PH1]]:
-; DEFAULT-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP1]], 2
+; DEFAULT-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP45:%.*]] = shl nuw i64 [[TMP9]], 2
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP45]]
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
@@ -1193,123 +1186,29 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP49:%.*]] = shl nuw nsw i64 [[TMP9]], 1
-; DEFAULT-NEXT: [[TMP53:%.*]] = mul nuw nsw i64 [[TMP9]], 3
-; DEFAULT-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP9]]
-; DEFAULT-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP49]]
-; DEFAULT-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP53]]
-; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x i8>, ptr [[TMP57]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 4 x i8>, ptr [[TMP61]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD8:%.*]] = load <vscale x 4 x i8>, ptr [[TMP65]], align 1
-; DEFAULT-NEXT: [[TMP11:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP69:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD6]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP73:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD7]] to <vscale x 4 x float>
+; DEFAULT-NEXT: [[WIDE_LOAD8:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
; DEFAULT-NEXT: [[TMP91:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD8]] to <vscale x 4 x float>
; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP9]]
-; DEFAULT-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP49]]
-; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP53]]
-; DEFAULT-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i8>, ptr [[TMP26]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD11:%.*]] = load <vscale x 4 x i8>, ptr [[TMP27]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD12:%.*]] = load <vscale x 4 x i8>, ptr [[TMP28]], align 1
-; DEFAULT-NEXT: [[TMP13:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD5]], zeroinitializer
-; DEFAULT-NEXT: [[TMP30:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD10]], zeroinitializer
-; DEFAULT-NEXT: [[TMP31:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD11]], zeroinitializer
+; DEFAULT-NEXT: [[WIDE_LOAD12:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
; DEFAULT-NEXT: [[TMP32:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD12]], zeroinitializer
-; DEFAULT-NEXT: [[TMP14:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP34:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD6]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP35:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD7]], splat (i8 1)
; DEFAULT-NEXT: [[TMP36:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD8]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP14]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP13]])
-; DEFAULT-NEXT: [[TMP38:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP34]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP30]])
-; DEFAULT-NEXT: [[TMP39:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP35]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP31]])
; DEFAULT-NEXT: [[TMP40:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP36]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP32]])
-; DEFAULT-NEXT: [[TMP17:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP16]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP42:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP38]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP43:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP39]], splat (i8 1)
; DEFAULT-NEXT: [[TMP44:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP40]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP18:%.*]] = select <vscale x 4 x i1> [[TMP17]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[TMP46:%.*]] = select <vscale x 4 x i1> [[TMP42]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[TMP47:%.*]] = select <vscale x 4 x i1> [[TMP43]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
; DEFAULT-NEXT: [[TMP48:%.*]] = select <vscale x 4 x i1> [[TMP44]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT: [[PREDPHI13:%.*]] = select <vscale x 4 x i1> [[TMP30]], <vscale x 4 x i32> [[TMP46]], <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT: [[PREDPHI14:%.*]] = select <vscale x 4 x i1> [[TMP31]], <vscale x 4 x i32> [[TMP47]], <vscale x 4 x i32> zeroinitializer
; DEFAULT-NEXT: [[PREDPHI15:%.*]] = select <vscale x 4 x i1> [[TMP32]], <vscale x 4 x i32> [[TMP48]], <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP19:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; DEFAULT-NEXT: [[TMP50:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD6]] to <vscale x 4 x i32>
-; DEFAULT-NEXT: [[TMP51:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD7]] to <vscale x 4 x i32>
; DEFAULT-NEXT: [[TMP52:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD8]] to <vscale x 4 x i32>
-; DEFAULT-NEXT: [[TMP20:%.*]] = sub <vscale x 4 x i32> [[PREDPHI]], [[TMP19]]
-; DEFAULT-NEXT: [[TMP54:%.*]] = sub <vscale x 4 x i32> [[PREDPHI13]], [[TMP50]]
-; DEFAULT-NEXT: [[TMP55:%.*]] = sub <vscale x 4 x i32> [[PREDPHI14]], [[TMP51]]
; DEFAULT-NEXT: [[TMP56:%.*]] = sub <vscale x 4 x i32> [[PREDPHI15]], [[TMP52]]
-; DEFAULT-NEXT: [[TMP21:%.*]] = sitofp <vscale x 4 x i32> [[TMP20]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP58:%.*]] = sitofp <vscale x 4 x i32> [[TMP54]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP59:%.*]] = sitofp <vscale x 4 x i32> [[TMP55]] to <vscale x 4 x float>
; DEFAULT-NEXT: [[TMP60:%.*]] = sitofp <vscale x 4 x i32> [[TMP56]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP22:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP21]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP11]])
-; DEFAULT-NEXT: [[TMP62:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP58]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP69]])
-; DEFAULT-NEXT: [[TMP63:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP59]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP73]])
; DEFAULT-NEXT: [[TMP64:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP60]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP91]])
-; DEFAULT-NEXT: [[TMP23:%.*]] = fptoui <vscale x 4 x float> [[TMP22]] to <vscale x 4 x i8>
-; DEFAULT-NEXT: [[TMP66:%.*]] = fptoui <vscale x 4 x float> [[TMP62]] to <vscale x 4 x i8>
-; DEFAULT-NEXT: [[TMP67:%.*]] = fptoui <vscale x 4 x float> [[TMP63]] to <vscale x 4 x i8>
; DEFAULT-NEXT: [[TMP68:%.*]] = fptoui <vscale x 4 x float> [[TMP64]] to <vscale x 4 x i8>
; DEFAULT-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[TMP24]], i64 [[TMP9]]
-; DEFAULT-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[TMP24]], i64 [[TMP49]]
-; DEFAULT-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[TMP24]], i64 [[TMP53]]
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP23]], ptr [[TMP24]], align 1
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP66]], ptr [[TMP70]], align 1
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP67]], ptr [[TMP71]], align 1
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP68]], ptr [[TMP72]], align 1
+; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP68]], ptr [[TMP24]], align 1
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP45]]
; DEFAULT-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; DEFAULT: [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP33]]
-; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF30:![0-9]+]]
-; DEFAULT: [[VEC_EPILOG_PH]]:
-; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
-; DEFAULT-NEXT: [[TMP74:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP75:%.*]] = shl nuw i64 [[TMP74]], 1
-; DEFAULT-NEXT: [[N_MOD_VF16:%.*]] = urem i64 [[TMP0]], [[TMP75]]
-; DEFAULT-NEXT: [[N_VEC17:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF16]]
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[Y]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT19:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT18]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
-; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
-; DEFAULT-NEXT: [[INDEX20:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT24:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX20]]
-; DEFAULT-NEXT: [[WIDE_LOAD21:%.*]] = load <vscale x 2 x i8>, ptr [[TMP76]], align 1
-; DEFAULT-NEXT: [[TMP77:%.*]] = uitofp <vscale x 2 x i8> [[WIDE_LOAD21]] to <vscale x 2 x float>
-; DEFAULT-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX20]]
-; DEFAULT-NEXT: [[WIDE_LOAD22:%.*]] = load <vscale x 2 x i8>, ptr [[TMP78]], align 1
-; DEFAULT-NEXT: [[TMP79:%.*]] = icmp ne <vscale x 2 x i8> [[WIDE_LOAD22]], zeroinitializer
-; DEFAULT-NEXT: [[TMP80:%.*]] = xor <vscale x 2 x i8> [[WIDE_LOAD21]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP81:%.*]] = call <vscale x 2 x i8> @llvm.masked.udiv.nxv2i8(<vscale x 2 x i8> [[TMP80]], <vscale x 2 x i8> [[BROADCAST_SPLAT19]], <vscale x 2 x i1> [[TMP79]])
-; DEFAULT-NEXT: [[TMP82:%.*]] = icmp ugt <vscale x 2 x i8> [[TMP81]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP83:%.*]] = select <vscale x 2 x i1> [[TMP82]], <vscale x 2 x i32> zeroinitializer, <vscale x 2 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[PREDPHI23:%.*]] = select <vscale x 2 x i1> [[TMP79]], <vscale x 2 x i32> [[TMP83]], <vscale x 2 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP84:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD21]] to <vscale x 2 x i32>
-; DEFAULT-NEXT: [[TMP85:%.*]] = sub <vscale x 2 x i32> [[PREDPHI23]], [[TMP84]]
-; DEFAULT-NEXT: [[TMP86:%.*]] = sitofp <vscale x 2 x i32> [[TMP85]] to <vscale x 2 x float>
-; DEFAULT-NEXT: [[TMP87:%.*]] = call <vscale x 2 x float> @llvm.fmuladd.nxv2f32(<vscale x 2 x float> [[TMP86]], <vscale x 2 x float> splat (float 3.000000e+00), <vscale x 2 x float> [[TMP77]])
-; DEFAULT-NEXT: [[TMP88:%.*]] = fptoui <vscale x 2 x float> [[TMP87]] to <vscale x 2 x i8>
-; DEFAULT-NEXT: [[TMP89:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX20]]
-; DEFAULT-NEXT: store <vscale x 2 x i8> [[TMP88]], ptr [[TMP89]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT24]] = add nuw i64 [[INDEX20]], [[TMP75]]
-; DEFAULT-NEXT: [[TMP90:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC17]]
-; DEFAULT-NEXT: br i1 [[TMP90]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[TMP25]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[CMP_N25:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC17]]
-; DEFAULT-NEXT: br i1 [[CMP_N25]], [[EXIT]], label %[[SCALAR_PH]]
+; DEFAULT-NEXT: [[CMP_N25:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[CMP_N25]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
;
; PRED-LABEL: define void @pred_udiv_select_cost(
@@ -1346,18 +1245,18 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; PRED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr align 1 [[TMP14]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
; PRED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_MASKED_LOAD5]], zeroinitializer
; PRED-NEXT: [[TMP11:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i1> zeroinitializer
-; PRED-NEXT: [[TMP22:%.*]] = xor <vscale x 4 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1)
-; PRED-NEXT: [[TMP23:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP22]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP11]])
-; PRED-NEXT: [[TMP15:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP23]], splat (i8 1)
-; PRED-NEXT: [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; PRED-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[TMP16]], <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT: [[TMP17:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
-; PRED-NEXT: [[TMP18:%.*]] = sub <vscale x 4 x i32> [[PREDPHI]], [[TMP17]]
-; PRED-NEXT: [[TMP19:%.*]] = sitofp <vscale x 4 x i32> [[TMP18]] to <vscale x 4 x float>
-; PRED-NEXT: [[TMP20:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP19]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP9]])
-; PRED-NEXT: [[TMP21:%.*]] = fptoui <vscale x 4 x float> [[TMP20]] to <vscale x 4 x i8>
+; PRED-NEXT: [[TMP21:%.*]] = xor <vscale x 4 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1)
+; PRED-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP21]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP11]])
+; PRED-NEXT: [[TMP22:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP13]], splat (i8 1)
+; PRED-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP22]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
+; PRED-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT: [[TMP16:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
+; PRED-NEXT: [[TMP17:%.*]] = sub <vscale x 4 x i32> [[PREDPHI]], [[TMP16]]
+; PRED-NEXT: [[TMP18:%.*]] = sitofp <vscale x 4 x i32> [[TMP17]] to <vscale x 4 x float>
+; PRED-NEXT: [[TMP19:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP18]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP9]])
+; PRED-NEXT: [[TMP20:%.*]] = fptoui <vscale x 4 x float> [[TMP19]] to <vscale x 4 x i8>
; PRED-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
-; PRED-NEXT: call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP21]], ptr align 1 [[TMP27]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; PRED-NEXT: call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP20]], ptr align 1 [[TMP27]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
; PRED-NEXT: [[TMP28:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
index 556ba7c0ba08a..c71139e93caab 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
@@ -205,7 +205,7 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = not vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = not vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP15:%[0-9]+]]> = vector-pointer vp<%next.gep>
; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP15]]>, ir<1>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP16:%[0-9]+]]> = vector-pointer vp<%next.gep>
@@ -236,7 +236,7 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP11]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP12]]> = or vp<[[VP9]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = or vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP14]]> = not vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP14]]> = not vp<[[VP13]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP15]]> = vector-pointer vp<%next.gep>
; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP15]]>, ir<1>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP16]]> = vector-pointer vp<%next.gep>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
index bba6e0c81fbb2..309e9a90a3c99 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
@@ -1149,52 +1149,10 @@ exit:
define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end, i64 %x) {
; COST-LABEL: define void @br_under_switch_default_common_dest_with_case(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
-; COST-NEXT: [[ENTRY:.*]]:
-; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
-; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
-; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
-; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
-; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
-; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
-; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COST: [[VECTOR_PH]]:
-; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
-; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; COST-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
-; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
-; COST-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
-; COST-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COST-NEXT: br label %[[VECTOR_BODY:.*]]
-; COST: [[VECTOR_BODY]]:
-; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 3
-; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
-; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
-; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
-; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
-; COST-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP7]], [[TMP8]]
-; COST-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
-; COST-NEXT: [[TMP11:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; COST-NEXT: [[TMP12:%.*]] = xor <4 x i1> [[TMP11]], splat (i1 true)
-; COST-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
-; COST-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP13]], [[TMP8]]
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]])
-; COST-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]])
-; COST-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP10]]
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP16]])
-; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COST-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; COST: [[MIDDLE_BLOCK]]:
-; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COST: [[SCALAR_PH]]:
-; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
-; COST-NEXT: br label %[[LOOP_HEADER:.*]]
-; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[SCALAR_PH:.*]]:
+; COST-NEXT: br label %[[LOOP_HEADER1:.*]]
+; COST: [[LOOP_HEADER1]]:
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -1216,7 +1174,7 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER1]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
More information about the llvm-commits
mailing list