[llvm] [LV] Add cost for VPInstruction::Not. (PR #198445)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Sun May 31 18:38:39 PDT 2026
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/198445
>From 037a2e347818bc5fec631ab88f44e65e54c9e33b Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 18 May 2026 19:54:38 -0700
Subject: [PATCH 1/6] [LV] Add cost for VPInstruction::Not.
This patch add the cost for VPInstruction::Now which will lower to Xor.
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +
.../LoopVectorize/AArch64/call-costs.ll | 85 ++-------
.../LoopVectorize/AArch64/cmp_cost.ll | 6 +-
.../AArch64/conditional-branches-cost.ll | 173 ++++--------------
.../AArch64/induction-costs-sve.ll | 32 ++--
.../LoopVectorize/AArch64/optsize_minsize.ll | 53 ++----
.../X86/CostModel/vpinstruction-cost.ll | 4 +-
.../LoopVectorize/X86/predicate-switch.ll | 52 +-----
8 files changed, 102 insertions(+), 310 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 59084bbaf41b3..224d3e7499494 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1402,6 +1402,13 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
VecTy, Ctx.CostKind, 0);
}
+ case VPInstruction::Not: {
+ Type *RetTy = Ctx.Types.inferScalarType(getOperand(0));
+ if (!vputils::onlyFirstLaneUsed(this))
+ RetTy = toVectorTy(RetTy, VF);
+ return Ctx.TTI.getArithmeticInstrCost(Instruction::Xor, RetTy,
+ Ctx.CostKind);
+ }
case Instruction::FCmp:
case Instruction::ICmp: {
// FIXME: We don't handle scalar compares inside the loop here yet, as loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index 1ca68083d58ad..2587463d77f63 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -104,77 +104,26 @@ exit:
define void @call_scalarized(ptr noalias %src, ptr noalias %dst) {
; CHECK-LABEL: define void @call_scalarized(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[LOOP_HEADER:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = sub i64 100, [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[TMP2]], i64 -1
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP2]], i64 -3
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <2 x double> [[WIDE_LOAD]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <2 x double> [[WIDE_LOAD1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[TMP5:%.*]] = fcmp une <2 x double> [[REVERSE]], splat (double 4.000000e+00)
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp une <2 x double> [[REVERSE2]], splat (double 4.000000e+00)
-; CHECK-NEXT: [[TMP7:%.*]] = fcmp ugt <2 x double> [[REVERSE]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp ugt <2 x double> [[REVERSE2]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i1> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: [[TMP10:%.*]] = or <2 x i1> [[TMP6]], [[TMP8]]
-; CHECK-NEXT: [[TMP11:%.*]] = xor <2 x i1> [[TMP9]], splat (i1 true)
-; CHECK-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[TMP10]], splat (i1 true)
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP11]], i64 0
-; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[REVERSE]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = call double @llvm.sqrt.f64(double [[TMP15]])
-; CHECK-NEXT: store double [[TMP16]], ptr [[TMP14]], align 8
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP11]], i64 1
-; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; CHECK: [[PRED_STORE_IF3]]:
-; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP0]], -1
-; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], -1
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <2 x double> [[REVERSE]], i64 1
-; CHECK-NEXT: [[TMP22:%.*]] = call double @llvm.sqrt.f64(double [[TMP21]])
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP20]], align 8
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[LOOP_HEADER]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_IF3:.*]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT: [[CMP295:%.*]] = fcmp une double [[TMP24]], 4.000000e+00
+; CHECK-NEXT: [[CMP299:%.*]] = fcmp ugt double [[TMP24]], 0.000000e+00
+; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[CMP295]], [[CMP299]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF3]], label %[[PRED_STORE_CONTINUE4:.*]]
; CHECK: [[PRED_STORE_CONTINUE4]]:
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP12]], i64 0
-; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP0]], -2
-; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[TMP24]], -1
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP25]]
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <2 x double> [[REVERSE2]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = call double @llvm.sqrt.f64(double [[TMP27]])
-; CHECK-NEXT: store double [[TMP28]], ptr [[TMP26]], align 8
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <2 x i1> [[TMP12]], i64 1
-; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[TMP0]], -3
-; CHECK-NEXT: [[TMP31:%.*]] = add i64 [[TMP30]], -1
-; CHECK-NEXT: [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = extractelement <2 x double> [[REVERSE2]], i64 1
-; CHECK-NEXT: [[TMP34:%.*]] = call double @llvm.sqrt.f64(double [[TMP33]])
-; CHECK-NEXT: store double [[TMP34]], ptr [[TMP32]], align 8
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP25:%.*]] = call double @llvm.sqrt.f64(double [[TMP24]])
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV_NEXT]]
+; CHECK-NEXT: store double [[TMP25]], ptr [[TMP23]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_IF3]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
index ff79ae808abcd..d1f9381325603 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
@@ -39,7 +39,7 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
@@ -71,7 +71,7 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 4: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
@@ -124,7 +124,7 @@ define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi double [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 2c52d8e499a8b..21e9c3a9d0ec2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1162,28 +1162,21 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 1
+; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 8)
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; DEFAULT-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP33:%.*]] = shl nuw i64 [[TMP29]], 1
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; DEFAULT-NEXT: [[TMP37:%.*]] = mul i64 [[TMP5]], 4
; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[C1]], [[A2]]
-; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP37]]
-; DEFAULT-NEXT: [[TMP41:%.*]] = mul i64 [[TMP5]], 4
+; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
; DEFAULT-NEXT: [[TMP7:%.*]] = sub i64 [[C1]], [[B3]]
-; DEFAULT-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP7]], [[TMP41]]
+; DEFAULT-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
; DEFAULT-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; DEFAULT-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[TMP15:%.*]] = shl nuw i64 [[TMP1]], 4
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP0]], [[TMP15]]
-; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK5]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; DEFAULT-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
; DEFAULT: [[VECTOR_PH1]]:
-; DEFAULT-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP1]], 2
+; DEFAULT-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP45:%.*]] = shl nuw i64 [[TMP9]], 2
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP45]]
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
@@ -1193,123 +1186,29 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP49:%.*]] = shl nuw nsw i64 [[TMP9]], 1
-; DEFAULT-NEXT: [[TMP53:%.*]] = mul nuw nsw i64 [[TMP9]], 3
-; DEFAULT-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP9]]
-; DEFAULT-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP49]]
-; DEFAULT-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP53]]
-; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x i8>, ptr [[TMP57]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 4 x i8>, ptr [[TMP61]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD8:%.*]] = load <vscale x 4 x i8>, ptr [[TMP65]], align 1
-; DEFAULT-NEXT: [[TMP11:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP69:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD6]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP73:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD7]] to <vscale x 4 x float>
+; DEFAULT-NEXT: [[WIDE_LOAD8:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
; DEFAULT-NEXT: [[TMP91:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD8]] to <vscale x 4 x float>
; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP9]]
-; DEFAULT-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP49]]
-; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP53]]
-; DEFAULT-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i8>, ptr [[TMP26]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD11:%.*]] = load <vscale x 4 x i8>, ptr [[TMP27]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD12:%.*]] = load <vscale x 4 x i8>, ptr [[TMP28]], align 1
-; DEFAULT-NEXT: [[TMP13:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD5]], zeroinitializer
-; DEFAULT-NEXT: [[TMP30:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD10]], zeroinitializer
-; DEFAULT-NEXT: [[TMP31:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD11]], zeroinitializer
+; DEFAULT-NEXT: [[WIDE_LOAD12:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
; DEFAULT-NEXT: [[TMP32:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_LOAD12]], zeroinitializer
-; DEFAULT-NEXT: [[TMP14:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP34:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD6]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP35:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD7]], splat (i8 1)
; DEFAULT-NEXT: [[TMP36:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD8]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP14]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP13]])
-; DEFAULT-NEXT: [[TMP38:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP34]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP30]])
-; DEFAULT-NEXT: [[TMP39:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP35]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP31]])
; DEFAULT-NEXT: [[TMP40:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP36]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP32]])
-; DEFAULT-NEXT: [[TMP17:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP16]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP42:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP38]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP43:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP39]], splat (i8 1)
; DEFAULT-NEXT: [[TMP44:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP40]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP18:%.*]] = select <vscale x 4 x i1> [[TMP17]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[TMP46:%.*]] = select <vscale x 4 x i1> [[TMP42]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[TMP47:%.*]] = select <vscale x 4 x i1> [[TMP43]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
; DEFAULT-NEXT: [[TMP48:%.*]] = select <vscale x 4 x i1> [[TMP44]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT: [[PREDPHI13:%.*]] = select <vscale x 4 x i1> [[TMP30]], <vscale x 4 x i32> [[TMP46]], <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT: [[PREDPHI14:%.*]] = select <vscale x 4 x i1> [[TMP31]], <vscale x 4 x i32> [[TMP47]], <vscale x 4 x i32> zeroinitializer
; DEFAULT-NEXT: [[PREDPHI15:%.*]] = select <vscale x 4 x i1> [[TMP32]], <vscale x 4 x i32> [[TMP48]], <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP19:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; DEFAULT-NEXT: [[TMP50:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD6]] to <vscale x 4 x i32>
-; DEFAULT-NEXT: [[TMP51:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD7]] to <vscale x 4 x i32>
; DEFAULT-NEXT: [[TMP52:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD8]] to <vscale x 4 x i32>
-; DEFAULT-NEXT: [[TMP20:%.*]] = sub <vscale x 4 x i32> [[PREDPHI]], [[TMP19]]
-; DEFAULT-NEXT: [[TMP54:%.*]] = sub <vscale x 4 x i32> [[PREDPHI13]], [[TMP50]]
-; DEFAULT-NEXT: [[TMP55:%.*]] = sub <vscale x 4 x i32> [[PREDPHI14]], [[TMP51]]
; DEFAULT-NEXT: [[TMP56:%.*]] = sub <vscale x 4 x i32> [[PREDPHI15]], [[TMP52]]
-; DEFAULT-NEXT: [[TMP21:%.*]] = sitofp <vscale x 4 x i32> [[TMP20]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP58:%.*]] = sitofp <vscale x 4 x i32> [[TMP54]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP59:%.*]] = sitofp <vscale x 4 x i32> [[TMP55]] to <vscale x 4 x float>
; DEFAULT-NEXT: [[TMP60:%.*]] = sitofp <vscale x 4 x i32> [[TMP56]] to <vscale x 4 x float>
-; DEFAULT-NEXT: [[TMP22:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP21]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP11]])
-; DEFAULT-NEXT: [[TMP62:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP58]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP69]])
-; DEFAULT-NEXT: [[TMP63:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP59]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP73]])
; DEFAULT-NEXT: [[TMP64:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP60]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP91]])
-; DEFAULT-NEXT: [[TMP23:%.*]] = fptoui <vscale x 4 x float> [[TMP22]] to <vscale x 4 x i8>
-; DEFAULT-NEXT: [[TMP66:%.*]] = fptoui <vscale x 4 x float> [[TMP62]] to <vscale x 4 x i8>
-; DEFAULT-NEXT: [[TMP67:%.*]] = fptoui <vscale x 4 x float> [[TMP63]] to <vscale x 4 x i8>
; DEFAULT-NEXT: [[TMP68:%.*]] = fptoui <vscale x 4 x float> [[TMP64]] to <vscale x 4 x i8>
; DEFAULT-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[TMP24]], i64 [[TMP9]]
-; DEFAULT-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[TMP24]], i64 [[TMP49]]
-; DEFAULT-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[TMP24]], i64 [[TMP53]]
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP23]], ptr [[TMP24]], align 1
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP66]], ptr [[TMP70]], align 1
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP67]], ptr [[TMP71]], align 1
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP68]], ptr [[TMP72]], align 1
+; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP68]], ptr [[TMP24]], align 1
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP45]]
; DEFAULT-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; DEFAULT: [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP33]]
-; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF30:![0-9]+]]
-; DEFAULT: [[VEC_EPILOG_PH]]:
-; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
-; DEFAULT-NEXT: [[TMP74:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP75:%.*]] = shl nuw i64 [[TMP74]], 1
-; DEFAULT-NEXT: [[N_MOD_VF16:%.*]] = urem i64 [[TMP0]], [[TMP75]]
-; DEFAULT-NEXT: [[N_VEC17:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF16]]
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[Y]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT19:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT18]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
-; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
-; DEFAULT-NEXT: [[INDEX20:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT24:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX20]]
-; DEFAULT-NEXT: [[WIDE_LOAD21:%.*]] = load <vscale x 2 x i8>, ptr [[TMP76]], align 1
-; DEFAULT-NEXT: [[TMP77:%.*]] = uitofp <vscale x 2 x i8> [[WIDE_LOAD21]] to <vscale x 2 x float>
-; DEFAULT-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX20]]
-; DEFAULT-NEXT: [[WIDE_LOAD22:%.*]] = load <vscale x 2 x i8>, ptr [[TMP78]], align 1
-; DEFAULT-NEXT: [[TMP79:%.*]] = icmp ne <vscale x 2 x i8> [[WIDE_LOAD22]], zeroinitializer
-; DEFAULT-NEXT: [[TMP80:%.*]] = xor <vscale x 2 x i8> [[WIDE_LOAD21]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP81:%.*]] = call <vscale x 2 x i8> @llvm.masked.udiv.nxv2i8(<vscale x 2 x i8> [[TMP80]], <vscale x 2 x i8> [[BROADCAST_SPLAT19]], <vscale x 2 x i1> [[TMP79]])
-; DEFAULT-NEXT: [[TMP82:%.*]] = icmp ugt <vscale x 2 x i8> [[TMP81]], splat (i8 1)
-; DEFAULT-NEXT: [[TMP83:%.*]] = select <vscale x 2 x i1> [[TMP82]], <vscale x 2 x i32> zeroinitializer, <vscale x 2 x i32> splat (i32 255)
-; DEFAULT-NEXT: [[PREDPHI23:%.*]] = select <vscale x 2 x i1> [[TMP79]], <vscale x 2 x i32> [[TMP83]], <vscale x 2 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP84:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD21]] to <vscale x 2 x i32>
-; DEFAULT-NEXT: [[TMP85:%.*]] = sub <vscale x 2 x i32> [[PREDPHI23]], [[TMP84]]
-; DEFAULT-NEXT: [[TMP86:%.*]] = sitofp <vscale x 2 x i32> [[TMP85]] to <vscale x 2 x float>
-; DEFAULT-NEXT: [[TMP87:%.*]] = call <vscale x 2 x float> @llvm.fmuladd.nxv2f32(<vscale x 2 x float> [[TMP86]], <vscale x 2 x float> splat (float 3.000000e+00), <vscale x 2 x float> [[TMP77]])
-; DEFAULT-NEXT: [[TMP88:%.*]] = fptoui <vscale x 2 x float> [[TMP87]] to <vscale x 2 x i8>
-; DEFAULT-NEXT: [[TMP89:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX20]]
-; DEFAULT-NEXT: store <vscale x 2 x i8> [[TMP88]], ptr [[TMP89]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT24]] = add nuw i64 [[INDEX20]], [[TMP75]]
-; DEFAULT-NEXT: [[TMP90:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC17]]
-; DEFAULT-NEXT: br i1 [[TMP90]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[TMP25]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[CMP_N25:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC17]]
-; DEFAULT-NEXT: br i1 [[CMP_N25]], [[EXIT]], label %[[SCALAR_PH]]
+; DEFAULT-NEXT: [[CMP_N25:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[CMP_N25]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
;
; PRED-LABEL: define void @pred_udiv_select_cost(
@@ -1322,7 +1221,7 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[C1]], [[A2]]
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
; PRED-NEXT: [[TMP4:%.*]] = sub i64 [[C1]], [[B3]]
@@ -1331,36 +1230,36 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; PRED-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[Y]], i64 0
-; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 4
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP0]])
+; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[Y]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PRED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr align 1 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
-; PRED-NEXT: [[TMP9:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x float>
+; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; PRED-NEXT: [[TMP9:%.*]] = uitofp <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x float>
; PRED-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr align 1 [[TMP14]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
-; PRED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_MASKED_LOAD5]], zeroinitializer
-; PRED-NEXT: [[TMP11:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i1> zeroinitializer
-; PRED-NEXT: [[TMP21:%.*]] = xor <vscale x 4 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1)
-; PRED-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP21]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP11]])
-; PRED-NEXT: [[TMP22:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP13]], splat (i8 1)
-; PRED-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP22]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
-; PRED-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT: [[TMP16:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
-; PRED-NEXT: [[TMP17:%.*]] = sub <vscale x 4 x i32> [[PREDPHI]], [[TMP16]]
-; PRED-NEXT: [[TMP18:%.*]] = sitofp <vscale x 4 x i32> [[TMP17]] to <vscale x 4 x float>
-; PRED-NEXT: [[TMP19:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP18]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP9]])
-; PRED-NEXT: [[TMP20:%.*]] = fptoui <vscale x 4 x float> [[TMP19]] to <vscale x 4 x i8>
+; PRED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP14]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; PRED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], zeroinitializer
+; PRED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i1> zeroinitializer
+; PRED-NEXT: [[TMP21:%.*]] = xor <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1)
+; PRED-NEXT: [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.masked.udiv.nxv16i8(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x i8> [[BROADCAST_SPLAT]], <vscale x 16 x i1> [[TMP11]])
+; PRED-NEXT: [[TMP22:%.*]] = icmp ugt <vscale x 16 x i8> [[TMP13]], splat (i8 1)
+; PRED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP22]], <vscale x 16 x i32> zeroinitializer, <vscale x 16 x i32> splat (i32 255)
+; PRED-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
+; PRED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; PRED-NEXT: [[TMP17:%.*]] = sub <vscale x 16 x i32> [[PREDPHI]], [[TMP16]]
+; PRED-NEXT: [[TMP18:%.*]] = sitofp <vscale x 16 x i32> [[TMP17]] to <vscale x 16 x float>
+; PRED-NEXT: [[TMP19:%.*]] = call <vscale x 16 x float> @llvm.fmuladd.nxv16f32(<vscale x 16 x float> [[TMP18]], <vscale x 16 x float> splat (float 3.000000e+00), <vscale x 16 x float> [[TMP9]])
+; PRED-NEXT: [[TMP20:%.*]] = fptoui <vscale x 16 x float> [[TMP19]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
-; PRED-NEXT: call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP20]], ptr align 1 [[TMP27]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; PRED-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP20]], ptr align 1 [[TMP27]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; PRED-NEXT: [[TMP28:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; PRED-NEXT: [[TMP28:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PRED-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
; PRED-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 8aca5b6022661..93a22a707cec2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -124,33 +124,33 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
; PRED-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3
-; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
-; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
+; PRED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 4
+; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[X]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP0]])
+; PRED-NEXT: [[TMP6:%.*]] = trunc <vscale x 16 x i32> [[BROADCAST_SPLAT]] to <vscale x 16 x i16>
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PRED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr align 1 [[TMP12]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i8> poison)
-; PRED-NEXT: [[TMP8:%.*]] = zext <vscale x 8 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 8 x i16>
-; PRED-NEXT: [[TMP9:%.*]] = mul <vscale x 8 x i16> [[TMP8]], [[TMP6]]
-; PRED-NEXT: [[TMP10:%.*]] = or <vscale x 8 x i16> [[TMP9]], [[TMP8]]
-; PRED-NEXT: [[TMP11:%.*]] = lshr <vscale x 8 x i16> [[TMP10]], splat (i16 1)
-; PRED-NEXT: [[TMP13:%.*]] = trunc <vscale x 8 x i16> [[TMP11]] to <vscale x 8 x i8>
+; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; PRED-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i16>
+; PRED-NEXT: [[TMP9:%.*]] = mul <vscale x 16 x i16> [[TMP8]], [[TMP6]]
+; PRED-NEXT: [[TMP10:%.*]] = or <vscale x 16 x i16> [[TMP9]], [[TMP8]]
+; PRED-NEXT: [[TMP11:%.*]] = lshr <vscale x 16 x i16> [[TMP10]], splat (i16 1)
+; PRED-NEXT: [[TMP13:%.*]] = trunc <vscale x 16 x i16> [[TMP11]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; PRED-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP13]], ptr align 1 [[TMP18]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; PRED-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP13]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
-; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; PRED-NEXT: [[TMP19:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; PRED-NEXT: [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PRED-NEXT: [[TMP20:%.*]] = xor i1 [[TMP19]], true
; PRED-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index 11370a93b2ffa..d3bcf7bb3c6cc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -480,44 +480,23 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
;
; MINSIZE-LABEL: define void @sve_tail_predicate_without_minsize(
; MINSIZE-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; MINSIZE-NEXT: [[ENTRY:.*:]]
-; MINSIZE-NEXT: br label %[[VECTOR_PH:.*]]
-; MINSIZE: [[VECTOR_PH]]:
-; MINSIZE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; MINSIZE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
-; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 15)
-; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[A]], i64 0
-; MINSIZE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[B]], i64 0
-; MINSIZE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[C]], i64 0
-; MINSIZE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-; MINSIZE-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
-; MINSIZE-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP6]] to i8
-; MINSIZE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP12]], i64 0
-; MINSIZE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[DOTSPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; MINSIZE-NEXT: [[VECTOR_PH:.*]]:
; MINSIZE-NEXT: br label %[[VECTOR_BODY:.*]]
; MINSIZE: [[VECTOR_BODY]]:
-; MINSIZE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MINSIZE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MINSIZE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i8> [ [[TMP10]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MINSIZE-NEXT: [[TMP15:%.*]] = mul <vscale x 16 x i8> [[BROADCAST_SPLAT]], [[VEC_IND]]
-; MINSIZE-NEXT: [[TMP16:%.*]] = lshr <vscale x 16 x i8> [[VEC_IND]], splat (i8 1)
-; MINSIZE-NEXT: [[TMP17:%.*]] = mul <vscale x 16 x i8> [[TMP16]], [[BROADCAST_SPLAT2]]
-; MINSIZE-NEXT: [[TMP18:%.*]] = add <vscale x 16 x i8> [[TMP17]], [[TMP15]]
-; MINSIZE-NEXT: [[TMP19:%.*]] = lshr <vscale x 16 x i8> [[VEC_IND]], splat (i8 2)
-; MINSIZE-NEXT: [[TMP20:%.*]] = mul <vscale x 16 x i8> [[TMP19]], [[BROADCAST_SPLAT4]]
-; MINSIZE-NEXT: [[TMP21:%.*]] = add <vscale x 16 x i8> [[TMP18]], [[TMP20]]
+; MINSIZE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MINSIZE-NEXT: [[TMP0:%.*]] = trunc nuw nsw i64 [[INDEX]] to i8
+; MINSIZE-NEXT: [[MUL:%.*]] = mul i8 [[A]], [[TMP0]]
+; MINSIZE-NEXT: [[SHR:%.*]] = lshr i8 [[TMP0]], 1
+; MINSIZE-NEXT: [[MUL5:%.*]] = mul i8 [[SHR]], [[B]]
+; MINSIZE-NEXT: [[ADD:%.*]] = add i8 [[MUL5]], [[MUL]]
+; MINSIZE-NEXT: [[SHR7:%.*]] = lshr i8 [[TMP0]], 2
+; MINSIZE-NEXT: [[MUL9:%.*]] = mul i8 [[SHR7]], [[C]]
+; MINSIZE-NEXT: [[ADD10:%.*]] = add i8 [[ADD]], [[MUL9]]
; MINSIZE-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
-; MINSIZE-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP21]], ptr align 1 [[TMP22]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; MINSIZE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 15)
-; MINSIZE-NEXT: [[TMP24:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; MINSIZE-NEXT: [[TMP23:%.*]] = xor i1 [[TMP24]], true
-; MINSIZE-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[DOTSPLAT]]
-; MINSIZE-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; MINSIZE: [[MIDDLE_BLOCK]]:
-; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; MINSIZE-NEXT: store i8 [[ADD10]], ptr [[TMP22]], align 1
+; MINSIZE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
+; MINSIZE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 15
+; MINSIZE-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]]
; MINSIZE: [[FOR_COND_CLEANUP]]:
; MINSIZE-NEXT: ret void
;
@@ -631,7 +610,7 @@ define void @dont_vectorize_with_minsize() {
; MINSIZE-NEXT: store <2 x i16> [[TMP9]], ptr [[TMP6]], align 2
; MINSIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; MINSIZE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; MINSIZE: [[MIDDLE_BLOCK]]:
; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; MINSIZE: [[FOR_COND_CLEANUP]]:
@@ -747,7 +726,7 @@ define void @vectorization_forced_minsize_reduce_width() {
; MINSIZE-NEXT: store <2 x i16> [[TMP9]], ptr [[TMP6]], align 2
; MINSIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; MINSIZE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; MINSIZE: [[MIDDLE_BLOCK]]:
; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; MINSIZE: [[FOR_COND_CLEANUP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
index e014d3e040d7d..cb43410f78746 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
@@ -210,7 +210,7 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = not vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = not vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP15:%[0-9]+]]> = vector-pointer vp<%next.gep>
; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP15]]>, ir<1>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP16:%[0-9]+]]> = vector-pointer vp<%next.gep>
@@ -242,7 +242,7 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP11]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP12]]> = or vp<[[VP9]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = or vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP14]]> = not vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP14]]> = not vp<[[VP13]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP15]]> = vector-pointer vp<%next.gep>
; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP15]]>, ir<1>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP16]]> = vector-pointer vp<%next.gep>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
index bba6e0c81fbb2..309e9a90a3c99 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
@@ -1149,52 +1149,10 @@ exit:
define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end, i64 %x) {
; COST-LABEL: define void @br_under_switch_default_common_dest_with_case(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
-; COST-NEXT: [[ENTRY:.*]]:
-; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
-; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
-; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
-; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
-; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
-; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
-; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COST: [[VECTOR_PH]]:
-; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
-; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; COST-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
-; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
-; COST-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
-; COST-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COST-NEXT: br label %[[VECTOR_BODY:.*]]
-; COST: [[VECTOR_BODY]]:
-; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 3
-; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
-; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
-; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
-; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
-; COST-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP7]], [[TMP8]]
-; COST-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
-; COST-NEXT: [[TMP11:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; COST-NEXT: [[TMP12:%.*]] = xor <4 x i1> [[TMP11]], splat (i1 true)
-; COST-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
-; COST-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP13]], [[TMP8]]
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]])
-; COST-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]])
-; COST-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP10]]
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP16]])
-; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COST-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; COST: [[MIDDLE_BLOCK]]:
-; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COST: [[SCALAR_PH]]:
-; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
-; COST-NEXT: br label %[[LOOP_HEADER:.*]]
-; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[SCALAR_PH:.*]]:
+; COST-NEXT: br label %[[LOOP_HEADER1:.*]]
+; COST: [[LOOP_HEADER1]]:
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -1216,7 +1174,7 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER1]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
>From bad1769cf11ff196dff5c7cefb64a9428bc07817 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 19 May 2026 17:27:58 -0700
Subject: [PATCH 2/6] !fixup, return free cost when Not can be folded into
branch.
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 18 ++--
.../LoopVectorize/AArch64/call-costs.ll | 94 +++++++++++++++----
.../AArch64/conditional-branches-cost.ll | 48 +++++-----
.../AArch64/induction-costs-sve.ll | 32 +++----
.../LoopVectorize/AArch64/optsize_minsize.ll | 53 +++++++----
5 files changed, 162 insertions(+), 83 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 224d3e7499494..102b7f41429e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1066,6 +1066,9 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
Type *ScalarTy = Ctx.Types.inferScalarType(this);
Type *ResultTy = VF.isVector() ? toVectorTy(ScalarTy, VF) : ScalarTy;
switch (Opcode) {
+ case VPInstruction::Not:
+ return Ctx.TTI.getArithmeticInstrCost(Instruction::Xor, ResultTy,
+ Ctx.CostKind);
case Instruction::FNeg:
return Ctx.TTI.getArithmeticInstrCost(Opcode, ResultTy, Ctx.CostKind);
case Instruction::UDiv:
@@ -1402,13 +1405,14 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
VecTy, Ctx.CostKind, 0);
}
- case VPInstruction::Not: {
- Type *RetTy = Ctx.Types.inferScalarType(getOperand(0));
- if (!vputils::onlyFirstLaneUsed(this))
- RetTy = toVectorTy(RetTy, VF);
- return Ctx.TTI.getArithmeticInstrCost(Instruction::Xor, RetTy,
- Ctx.CostKind);
- }
+ case VPInstruction::Not:
+ // InstCombine will fold `xor` to the conditional branch.
+ if (hasOneUse() && match(const_cast<VPUser *>(getSingleUser()),
+ m_BranchOnCond(m_VPValue())))
+ return 0;
+ return getCostForRecipeWithOpcode(
+ getOpcode(),
+ vputils::onlyFirstLaneUsed(this) ? ElementCount::getFixed(1) : VF, Ctx);
case Instruction::FCmp:
case Instruction::ICmp: {
// FIXME: We don't handle scalar compares inside the loop here yet, as loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index 2587463d77f63..ef6fc9779c685 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -104,26 +104,77 @@ exit:
define void @call_scalarized(ptr noalias %src, ptr noalias %dst) {
; CHECK-LABEL: define void @call_scalarized(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
-; CHECK-NEXT: [[LOOP_HEADER:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[LOOP_HEADER]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_IF3:.*]] ]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV_NEXT]]
-; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[GEP_SRC]], align 8
-; CHECK-NEXT: [[CMP295:%.*]] = fcmp une double [[TMP24]], 4.000000e+00
-; CHECK-NEXT: [[CMP299:%.*]] = fcmp ugt double [[TMP24]], 0.000000e+00
-; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[CMP295]], [[CMP299]]
-; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF3]], label %[[PRED_STORE_CONTINUE4:.*]]
-; CHECK: [[PRED_STORE_CONTINUE4]]:
-; CHECK-NEXT: [[TMP25:%.*]] = call double @llvm.sqrt.f64(double [[TMP24]])
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV_NEXT]]
-; CHECK-NEXT: store double [[TMP25]], ptr [[TMP23]], align 8
-; CHECK-NEXT: br label %[[PRED_STORE_IF3]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 100, [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[TMP2]], i64 -1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP2]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <2 x double> [[WIDE_LOAD]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <2 x double> [[WIDE_LOAD1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp une <2 x double> [[REVERSE]], splat (double 4.000000e+00)
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp une <2 x double> [[REVERSE2]], splat (double 4.000000e+00)
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp ugt <2 x double> [[REVERSE]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fcmp ugt <2 x double> [[REVERSE2]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i1> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = or <2 x i1> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = xor <2 x i1> [[TMP9]], splat (i1 true)
+; CHECK-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[TMP10]], splat (i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP11]], i64 0
+; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[REVERSE]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = call double @llvm.sqrt.f64(double [[TMP15]])
+; CHECK-NEXT: store double [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP11]], i64 1
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; CHECK: [[PRED_STORE_IF3]]:
-; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
-; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP0]], -1
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], -1
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <2 x double> [[REVERSE]], i64 1
+; CHECK-NEXT: [[TMP22:%.*]] = call double @llvm.sqrt.f64(double [[TMP21]])
+; CHECK-NEXT: store double [[TMP22]], ptr [[TMP20]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP12]], i64 0
+; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP0]], -2
+; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[TMP24]], -1
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP25]]
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <2 x double> [[REVERSE2]], i64 0
+; CHECK-NEXT: [[TMP28:%.*]] = call double @llvm.sqrt.f64(double [[TMP27]])
+; CHECK-NEXT: store double [[TMP28]], ptr [[TMP26]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <2 x i1> [[TMP12]], i64 1
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[TMP0]], -3
+; CHECK-NEXT: [[TMP31:%.*]] = add i64 [[TMP30]], -1
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP31]]
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <2 x double> [[REVERSE2]], i64 1
+; CHECK-NEXT: [[TMP34:%.*]] = call double @llvm.sqrt.f64(double [[TMP33]])
+; CHECK-NEXT: store double [[TMP34]], ptr [[TMP32]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
@@ -147,7 +198,7 @@ then:
loop.latch:
%tobool.not = icmp eq i64 %iv.next, 0
- br i1 %tobool.not, label %exit, label %loop.header
+ br i1 %tobool.not, label %exit, label %loop.header, !llvm.loop !0
exit:
ret void
@@ -226,7 +277,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -245,7 +296,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: store i32 [[COND]], ptr [[DST_GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -269,3 +320,6 @@ loop:
exit:
ret void
}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 2}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 21e9c3a9d0ec2..37311a1e4aa8d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1221,7 +1221,7 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
+; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[C1]], [[A2]]
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
; PRED-NEXT: [[TMP4:%.*]] = sub i64 [[C1]], [[B3]]
@@ -1230,36 +1230,36 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; PRED-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 4
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[Y]], i64 0
-; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; PRED-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[Y]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PRED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; PRED-NEXT: [[TMP9:%.*]] = uitofp <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x float>
+; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr align 1 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; PRED-NEXT: [[TMP9:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x float>
; PRED-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP14]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; PRED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], zeroinitializer
-; PRED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i1> zeroinitializer
-; PRED-NEXT: [[TMP21:%.*]] = xor <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1)
-; PRED-NEXT: [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.masked.udiv.nxv16i8(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x i8> [[BROADCAST_SPLAT]], <vscale x 16 x i1> [[TMP11]])
-; PRED-NEXT: [[TMP22:%.*]] = icmp ugt <vscale x 16 x i8> [[TMP13]], splat (i8 1)
-; PRED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP22]], <vscale x 16 x i32> zeroinitializer, <vscale x 16 x i32> splat (i32 255)
-; PRED-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
-; PRED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; PRED-NEXT: [[TMP17:%.*]] = sub <vscale x 16 x i32> [[PREDPHI]], [[TMP16]]
-; PRED-NEXT: [[TMP18:%.*]] = sitofp <vscale x 16 x i32> [[TMP17]] to <vscale x 16 x float>
-; PRED-NEXT: [[TMP19:%.*]] = call <vscale x 16 x float> @llvm.fmuladd.nxv16f32(<vscale x 16 x float> [[TMP18]], <vscale x 16 x float> splat (float 3.000000e+00), <vscale x 16 x float> [[TMP9]])
-; PRED-NEXT: [[TMP20:%.*]] = fptoui <vscale x 16 x float> [[TMP19]] to <vscale x 16 x i8>
+; PRED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr align 1 [[TMP14]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; PRED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 4 x i8> [[WIDE_MASKED_LOAD5]], zeroinitializer
+; PRED-NEXT: [[TMP11:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i1> zeroinitializer
+; PRED-NEXT: [[TMP21:%.*]] = xor <vscale x 4 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1)
+; PRED-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.masked.udiv.nxv4i8(<vscale x 4 x i8> [[TMP21]], <vscale x 4 x i8> [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP11]])
+; PRED-NEXT: [[TMP22:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP13]], splat (i8 1)
+; PRED-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP22]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
+; PRED-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT: [[TMP16:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
+; PRED-NEXT: [[TMP17:%.*]] = sub <vscale x 4 x i32> [[PREDPHI]], [[TMP16]]
+; PRED-NEXT: [[TMP18:%.*]] = sitofp <vscale x 4 x i32> [[TMP17]] to <vscale x 4 x float>
+; PRED-NEXT: [[TMP19:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP18]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP9]])
+; PRED-NEXT: [[TMP20:%.*]] = fptoui <vscale x 4 x float> [[TMP19]] to <vscale x 4 x i8>
; PRED-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
-; PRED-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP20]], ptr align 1 [[TMP27]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; PRED-NEXT: call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP20]], ptr align 1 [[TMP27]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; PRED-NEXT: [[TMP28:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; PRED-NEXT: [[TMP28:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PRED-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
; PRED-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 93a22a707cec2..8aca5b6022661 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -124,33 +124,33 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
+; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
; PRED-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 4
-; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[X]], i64 0
-; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT: [[TMP6:%.*]] = trunc <vscale x 16 x i32> [[BROADCAST_SPLAT]] to <vscale x 16 x i16>
+; PRED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3
+; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
+; PRED-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PRED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; PRED-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i16>
-; PRED-NEXT: [[TMP9:%.*]] = mul <vscale x 16 x i16> [[TMP8]], [[TMP6]]
-; PRED-NEXT: [[TMP10:%.*]] = or <vscale x 16 x i16> [[TMP9]], [[TMP8]]
-; PRED-NEXT: [[TMP11:%.*]] = lshr <vscale x 16 x i16> [[TMP10]], splat (i16 1)
-; PRED-NEXT: [[TMP13:%.*]] = trunc <vscale x 16 x i16> [[TMP11]] to <vscale x 16 x i8>
+; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr align 1 [[TMP12]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i8> poison)
+; PRED-NEXT: [[TMP8:%.*]] = zext <vscale x 8 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 8 x i16>
+; PRED-NEXT: [[TMP9:%.*]] = mul <vscale x 8 x i16> [[TMP8]], [[TMP6]]
+; PRED-NEXT: [[TMP10:%.*]] = or <vscale x 8 x i16> [[TMP9]], [[TMP8]]
+; PRED-NEXT: [[TMP11:%.*]] = lshr <vscale x 8 x i16> [[TMP10]], splat (i16 1)
+; PRED-NEXT: [[TMP13:%.*]] = trunc <vscale x 8 x i16> [[TMP11]] to <vscale x 8 x i8>
; PRED-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; PRED-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP13]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; PRED-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP13]], ptr align 1 [[TMP18]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
-; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; PRED-NEXT: [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; PRED-NEXT: [[TMP19:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PRED-NEXT: [[TMP20:%.*]] = xor i1 [[TMP19]], true
; PRED-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index d3bcf7bb3c6cc..11370a93b2ffa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -480,23 +480,44 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
;
; MINSIZE-LABEL: define void @sve_tail_predicate_without_minsize(
; MINSIZE-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; MINSIZE-NEXT: [[VECTOR_PH:.*]]:
+; MINSIZE-NEXT: [[ENTRY:.*:]]
+; MINSIZE-NEXT: br label %[[VECTOR_PH:.*]]
+; MINSIZE: [[VECTOR_PH]]:
+; MINSIZE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; MINSIZE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 15)
+; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[A]], i64 0
+; MINSIZE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[B]], i64 0
+; MINSIZE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[C]], i64 0
+; MINSIZE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; MINSIZE-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
+; MINSIZE-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP6]] to i8
+; MINSIZE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP12]], i64 0
+; MINSIZE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[DOTSPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; MINSIZE-NEXT: br label %[[VECTOR_BODY:.*]]
; MINSIZE: [[VECTOR_BODY]]:
-; MINSIZE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MINSIZE-NEXT: [[TMP0:%.*]] = trunc nuw nsw i64 [[INDEX]] to i8
-; MINSIZE-NEXT: [[MUL:%.*]] = mul i8 [[A]], [[TMP0]]
-; MINSIZE-NEXT: [[SHR:%.*]] = lshr i8 [[TMP0]], 1
-; MINSIZE-NEXT: [[MUL5:%.*]] = mul i8 [[SHR]], [[B]]
-; MINSIZE-NEXT: [[ADD:%.*]] = add i8 [[MUL5]], [[MUL]]
-; MINSIZE-NEXT: [[SHR7:%.*]] = lshr i8 [[TMP0]], 2
-; MINSIZE-NEXT: [[MUL9:%.*]] = mul i8 [[SHR7]], [[C]]
-; MINSIZE-NEXT: [[ADD10:%.*]] = add i8 [[ADD]], [[MUL9]]
+; MINSIZE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MINSIZE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MINSIZE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i8> [ [[TMP10]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MINSIZE-NEXT: [[TMP15:%.*]] = mul <vscale x 16 x i8> [[BROADCAST_SPLAT]], [[VEC_IND]]
+; MINSIZE-NEXT: [[TMP16:%.*]] = lshr <vscale x 16 x i8> [[VEC_IND]], splat (i8 1)
+; MINSIZE-NEXT: [[TMP17:%.*]] = mul <vscale x 16 x i8> [[TMP16]], [[BROADCAST_SPLAT2]]
+; MINSIZE-NEXT: [[TMP18:%.*]] = add <vscale x 16 x i8> [[TMP17]], [[TMP15]]
+; MINSIZE-NEXT: [[TMP19:%.*]] = lshr <vscale x 16 x i8> [[VEC_IND]], splat (i8 2)
+; MINSIZE-NEXT: [[TMP20:%.*]] = mul <vscale x 16 x i8> [[TMP19]], [[BROADCAST_SPLAT4]]
+; MINSIZE-NEXT: [[TMP21:%.*]] = add <vscale x 16 x i8> [[TMP18]], [[TMP20]]
; MINSIZE-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
-; MINSIZE-NEXT: store i8 [[ADD10]], ptr [[TMP22]], align 1
-; MINSIZE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
-; MINSIZE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 15
-; MINSIZE-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]]
+; MINSIZE-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP21]], ptr align 1 [[TMP22]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; MINSIZE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 15)
+; MINSIZE-NEXT: [[TMP24:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; MINSIZE-NEXT: [[TMP23:%.*]] = xor i1 [[TMP24]], true
+; MINSIZE-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[DOTSPLAT]]
+; MINSIZE-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; MINSIZE: [[MIDDLE_BLOCK]]:
+; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; MINSIZE: [[FOR_COND_CLEANUP]]:
; MINSIZE-NEXT: ret void
;
@@ -610,7 +631,7 @@ define void @dont_vectorize_with_minsize() {
; MINSIZE-NEXT: store <2 x i16> [[TMP9]], ptr [[TMP6]], align 2
; MINSIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; MINSIZE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; MINSIZE: [[MIDDLE_BLOCK]]:
; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; MINSIZE: [[FOR_COND_CLEANUP]]:
@@ -726,7 +747,7 @@ define void @vectorization_forced_minsize_reduce_width() {
; MINSIZE-NEXT: store <2 x i16> [[TMP9]], ptr [[TMP6]], align 2
; MINSIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; MINSIZE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; MINSIZE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; MINSIZE: [[MIDDLE_BLOCK]]:
; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
; MINSIZE: [[FOR_COND_CLEANUP]]:
>From ce2acb4cde700ab41f9c963312e2c6eb2af2fac3 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Wed, 20 May 2026 18:43:29 -0700
Subject: [PATCH 3/6] Address comments.
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 102b7f41429e9..7617b71a13733 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1407,9 +1407,9 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
}
case VPInstruction::Not:
// InstCombine will fold `xor` to the conditional branch.
- if (hasOneUse() && match(const_cast<VPUser *>(getSingleUser()),
- m_BranchOnCond(m_VPValue())))
- return 0;
+ if (auto *U = const_cast<VPUser *>(getSingleUser()))
+ if (match(U, m_BranchOnCond(m_VPValue())))
+ return 0;
return getCostForRecipeWithOpcode(
getOpcode(),
vputils::onlyFirstLaneUsed(this) ? ElementCount::getFixed(1) : VF, Ctx);
>From af8db6e5f285c6e1a3953ea97cdc0e5c9c21cd39 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 21 May 2026 17:54:51 -0700
Subject: [PATCH 4/6] Address comments.
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 +-
.../AArch64/vpinstruction-not-cost.ll | 318 ++++++++++++++++++
2 files changed, 321 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7617b71a13733..239f50ed73dce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1066,9 +1066,6 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
Type *ScalarTy = Ctx.Types.inferScalarType(this);
Type *ResultTy = VF.isVector() ? toVectorTy(ScalarTy, VF) : ScalarTy;
switch (Opcode) {
- case VPInstruction::Not:
- return Ctx.TTI.getArithmeticInstrCost(Instruction::Xor, ResultTy,
- Ctx.CostKind);
case Instruction::FNeg:
return Ctx.TTI.getArithmeticInstrCost(Opcode, ResultTy, Ctx.CostKind);
case Instruction::UDiv:
@@ -1410,9 +1407,9 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
if (auto *U = const_cast<VPUser *>(getSingleUser()))
if (match(U, m_BranchOnCond(m_VPValue())))
return 0;
- return getCostForRecipeWithOpcode(
- getOpcode(),
- vputils::onlyFirstLaneUsed(this) ? ElementCount::getFixed(1) : VF, Ctx);
+ return Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Xor, toVectorTy(Ctx.Types.inferScalarType(this), VF),
+ Ctx.CostKind);
case Instruction::FCmp:
case Instruction::ICmp: {
// FIXME: We don't handle scalar compares inside the loop here yet, as loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
new file mode 100644
index 0000000000000..4b17b25c82abe
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of" --filter "Selecting VF" --filter-out-after "Selecting VF" --version 6
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s
+
+target triple = "aarch64-none-elf"
+
+define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
+; CHECK-LABEL: 'multi_user_cmp'
+; CHECK: Cost of 1 for VF 2: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 2: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK: Cost of 1 for VF 2: exit condition instruction %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 2: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%load1> = load vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cmp1>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: vector loop backedge
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12:%[0-9]+]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14:%[0-9]+]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK: Cost of 0 for VF 2: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
+; CHECK: Cost of 0 for VF 2: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
+; CHECK: Cost of 0 for VF 2: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 2: IR %load1 = load float, ptr %arrayidx, align 4
+; CHECK: Cost of 0 for VF 2: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
+; CHECK: Cost of 0 for VF 2: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+; CHECK: Cost of 0 for VF 2: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (or) vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11:%[0-9]+]]> = freeze vp<[[VP10]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (or) vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
+; CHECK: Cost of 0 for VF 2: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK: Cost of 0 for VF 2: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK: Cost of 0 for VF 2: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
+; CHECK: Cost of 0 for VF 2: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
+; CHECK: Cost of 1 for VF 4: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK: Cost of 1 for VF 4: exit condition instruction %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 4: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 0 for VF 4: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%load1> = load vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK: Cost of 0 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 4: vector loop backedge
+; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 4: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK: Cost of 0 for VF 4: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
+; CHECK: Cost of 0 for VF 4: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
+; CHECK: Cost of 0 for VF 4: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 4: IR %load1 = load float, ptr %arrayidx, align 4
+; CHECK: Cost of 0 for VF 4: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
+; CHECK: Cost of 0 for VF 4: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+; CHECK: Cost of 0 for VF 4: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+; CHECK: Cost of 0 for VF 4: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 4: IR %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<%cmp.n>
+; CHECK: Cost of 0 for VF 4: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK: Cost of 0 for VF 4: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK: Cost of 0 for VF 4: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
+; CHECK: Cost of 0 for VF 4: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
+; CHECK: Cost of 1 for VF 8: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 8: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK: Cost of 1 for VF 8: exit condition instruction %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 8: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 8: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 8: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 8: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 0 for VF 8: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; CHECK: Cost of 0 for VF 8: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%load1> = load vp<[[VP5]]>
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
+; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
+; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 8: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK: Cost of 0 for VF 8: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 8: vector loop backedge
+; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 8: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK: Cost of 0 for VF 8: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
+; CHECK: Cost of 0 for VF 8: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
+; CHECK: Cost of 0 for VF 8: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 8: IR %load1 = load float, ptr %arrayidx, align 4
+; CHECK: Cost of 0 for VF 8: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
+; CHECK: Cost of 0 for VF 8: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+; CHECK: Cost of 0 for VF 8: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+; CHECK: Cost of 0 for VF 8: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 8: IR %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
+; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 8: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 8: EMIT branch-on-cond vp<%cmp.n>
+; CHECK: Cost of 0 for VF 8: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK: Cost of 0 for VF 8: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK: Cost of 0 for VF 8: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
+; CHECK: Cost of 0 for VF 8: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
+; CHECK: Cost of 1 for VF 16: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 16: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK: Cost of 1 for VF 16: exit condition instruction %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 16: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 16: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 16: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 16: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 0 for VF 16: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; CHECK: Cost of 0 for VF 16: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%load1> = load vp<[[VP5]]>
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
+; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 16: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK: Cost of 0 for VF 16: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 16: vector loop backedge
+; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 16: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK: Cost of 0 for VF 16: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
+; CHECK: Cost of 0 for VF 16: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
+; CHECK: Cost of 0 for VF 16: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 16: IR %load1 = load float, ptr %arrayidx, align 4
+; CHECK: Cost of 0 for VF 16: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
+; CHECK: Cost of 0 for VF 16: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+; CHECK: Cost of 0 for VF 16: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+; CHECK: Cost of 0 for VF 16: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 16: IR %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 16: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 16: EMIT branch-on-cond vp<%cmp.n>
+; CHECK: Cost of 0 for VF 16: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK: Cost of 0 for VF 16: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK: Cost of 0 for VF 16: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
+; CHECK: Cost of 0 for VF 16: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
+; CHECK: Cost of 1 for VF 32: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 32: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK: Cost of 1 for VF 32: exit condition instruction %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 32: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 32: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 32: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 32: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 0 for VF 32: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; CHECK: Cost of 0 for VF 32: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK: Cost of 8 for VF 32: WIDEN ir<%load1> = load vp<[[VP5]]>
+; CHECK: Cost of 8 for VF 32: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
+; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
+; CHECK: Cost of 2 for VF 32: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 32: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK: Cost of 0 for VF 32: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 32: vector loop backedge
+; CHECK: Cost of 0 for VF 32: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 32: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 32: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 32: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK: Cost of 0 for VF 32: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
+; CHECK: Cost of 0 for VF 32: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
+; CHECK: Cost of 0 for VF 32: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 32: IR %load1 = load float, ptr %arrayidx, align 4
+; CHECK: Cost of 0 for VF 32: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
+; CHECK: Cost of 0 for VF 32: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+; CHECK: Cost of 0 for VF 32: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+; CHECK: Cost of 0 for VF 32: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 32: IR %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
+; CHECK: Cost of 2 for VF 32: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 32: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 32: EMIT branch-on-cond vp<%cmp.n>
+; CHECK: Cost of 0 for VF 32: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK: Cost of 0 for VF 32: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK: Cost of 0 for VF 32: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
+; CHECK: Cost of 0 for VF 32: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
+; CHECK: Cost of 1 for VF 64: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 64: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK: Cost of 1 for VF 64: exit condition instruction %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 64: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 64: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 64: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 64: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 0 for VF 64: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; CHECK: Cost of 0 for VF 64: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK: Cost of 16 for VF 64: WIDEN ir<%load1> = load vp<[[VP5]]>
+; CHECK: Cost of 16 for VF 64: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
+; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
+; CHECK: Cost of 4 for VF 64: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 64: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK: Cost of 0 for VF 64: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 64: vector loop backedge
+; CHECK: Cost of 0 for VF 64: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 64: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 64: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 64: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK: Cost of 0 for VF 64: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
+; CHECK: Cost of 0 for VF 64: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
+; CHECK: Cost of 0 for VF 64: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 64: IR %load1 = load float, ptr %arrayidx, align 4
+; CHECK: Cost of 0 for VF 64: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
+; CHECK: Cost of 0 for VF 64: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+; CHECK: Cost of 0 for VF 64: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+; CHECK: Cost of 0 for VF 64: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 64: IR %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
+; CHECK: Cost of 4 for VF 64: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 64: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 64: EMIT branch-on-cond vp<%cmp.n>
+; CHECK: Cost of 0 for VF 64: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK: Cost of 0 for VF 64: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK: Cost of 0 for VF 64: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
+; CHECK: Cost of 0 for VF 64: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
+; CHECK: Cost of 1 for VF 128: induction instruction %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 128: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK: Cost of 1 for VF 128: exit condition instruction %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 128: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 128: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 128: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 128: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 0 for VF 128: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; CHECK: Cost of 0 for VF 128: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK: Cost of 32 for VF 128: WIDEN ir<%load1> = load vp<[[VP5]]>
+; CHECK: Cost of 32 for VF 128: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
+; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
+; CHECK: Cost of 8 for VF 128: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 128: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK: Cost of 0 for VF 128: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 128: vector loop backedge
+; CHECK: Cost of 0 for VF 128: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 128: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 128: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 128: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK: Cost of 0 for VF 128: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
+; CHECK: Cost of 0 for VF 128: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
+; CHECK: Cost of 0 for VF 128: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK: Cost of 0 for VF 128: IR %load1 = load float, ptr %arrayidx, align 4
+; CHECK: Cost of 0 for VF 128: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
+; CHECK: Cost of 0 for VF 128: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+; CHECK: Cost of 0 for VF 128: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+; CHECK: Cost of 0 for VF 128: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK: Cost of 0 for VF 128: IR %ec = icmp eq i64 %iv.next, %n
+; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
+; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
+; CHECK: Cost of 8 for VF 128: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
+; CHECK: Cost of 1 for VF 128: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 128: EMIT branch-on-cond vp<%cmp.n>
+; CHECK: Cost of 0 for VF 128: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK: Cost of 0 for VF 128: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK: Cost of 0 for VF 128: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
+; CHECK: Cost of 0 for VF 128: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
+; CHECK: LV: Selecting VF: 16.
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ]
+ %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ]
+ %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+ %load1 = load float, ptr %arrayidx, align 4
+ %cmp1 = fcmp olt float %load1, 0.000000e+00
+ %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+ %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %0 = select i1 %.any.0.off0, i32 2, i32 3
+ %1 = select i1 %all.off, i32 1, i32 %0
+ ret i32 %1
+}
>From 439906100a3995605750b25776af23379a33c418 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Fri, 22 May 2026 01:30:11 -0700
Subject: [PATCH 5/6] Fixup, check onlyfirstLaneused().
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 11 +++++++----
.../LoopVectorize/AArch64/vpinstruction-not-cost.ll | 6 +++---
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 239f50ed73dce..9642b5e5c63b9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1402,14 +1402,17 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
VecTy, Ctx.CostKind, 0);
}
- case VPInstruction::Not:
+ case VPInstruction::Not: {
+ Type *ValTy = Ctx.Types.inferScalarType(this);
// InstCombine will fold `xor` to the conditional branch.
if (auto *U = const_cast<VPUser *>(getSingleUser()))
if (match(U, m_BranchOnCond(m_VPValue())))
return 0;
- return Ctx.TTI.getArithmeticInstrCost(
- Instruction::Xor, toVectorTy(Ctx.Types.inferScalarType(this), VF),
- Ctx.CostKind);
+ if (!vputils::onlyFirstLaneUsed(this))
+ ValTy = toVectorTy(ValTy, VF);
+ return Ctx.TTI.getArithmeticInstrCost(Instruction::Xor, ValTy,
+ Ctx.CostKind);
+ }
case Instruction::FCmp:
case Instruction::ICmp: {
// FIXME: We don't handle scalar compares inside the loop here yet, as loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
index 4b17b25c82abe..ec420dce138bb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
@@ -202,7 +202,7 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK: Cost of 0 for VF 32: IR %ec = icmp eq i64 %iv.next, %n
; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 2 for VF 32: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 32: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
; CHECK: Cost of 1 for VF 32: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
@@ -243,7 +243,7 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK: Cost of 0 for VF 64: IR %ec = icmp eq i64 %iv.next, %n
; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 4 for VF 64: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 64: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
; CHECK: Cost of 1 for VF 64: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
@@ -284,7 +284,7 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK: Cost of 0 for VF 128: IR %ec = icmp eq i64 %iv.next, %n
; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 8 for VF 128: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 128: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
; CHECK: Cost of 1 for VF 128: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
>From d62044994a54a5aabb01b9fdc9c1d6239fe82c0f Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Sun, 31 May 2026 18:37:39 -0700
Subject: [PATCH 6/6] Move tests to AArch64/arith-costs.ll (renamed from
binop-costs.ll).
---
.../{binop-costs.ll => arith-costs.ll} | 42 ++-
.../AArch64/vpinstruction-not-cost.ll | 318 ------------------
2 files changed, 41 insertions(+), 319 deletions(-)
rename llvm/test/Transforms/LoopVectorize/AArch64/{binop-costs.ll => arith-costs.ll} (61%)
delete mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arith-costs.ll
similarity index 61%
rename from llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
rename to llvm/test/Transforms/LoopVectorize/AArch64/arith-costs.ll
index e42090d556f3c..292acc91f0470 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/arith-costs.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "[Cc]ost.*(udiv|fneg|fmul)"
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "[Cc]ost.*(udiv|fneg|fmul|not)"
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize %s -S -o - 2>&1 | FileCheck %s
; REQUIRES: asserts
@@ -69,4 +69,44 @@ exit:
ret void
}
+define i32 @test_scalar_and_widen_not_cost(ptr readonly %a, i64 noundef %n) {
+; CHECK-LABEL: 'multi_user_cmp'
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP11:%[0-9]+]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 2 for VF 32: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 32: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 4 for VF 64: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 64: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 8 for VF 128: EMIT vp<[[VP7]]> = not ir<%cmp1>
+; CHECK: Cost of 1 for VF 128: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ]
+ %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ]
+ %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+ %load1 = load float, ptr %arrayidx, align 4
+ %cmp1 = fcmp olt float %load1, 0.000000e+00
+ %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
+ %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %0 = select i1 %.any.0.off0, i32 2, i32 3
+ %1 = select i1 %all.off, i32 1, i32 %0
+ ret i32 %1
+}
+
attributes #0 = { "target-cpu"="neoverse-v2" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
deleted file mode 100644
index ec420dce138bb..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vpinstruction-not-cost.ll
+++ /dev/null
@@ -1,318 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of" --filter "Selecting VF" --filter-out-after "Selecting VF" --version 6
-; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s
-
-target triple = "aarch64-none-elf"
-
-define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
-; CHECK-LABEL: 'multi_user_cmp'
-; CHECK: Cost of 1 for VF 2: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 2: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 2: exit condition instruction %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 2: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%load1> = load vp<[[VP5]]>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cmp1>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: vector loop backedge
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12:%[0-9]+]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14:%[0-9]+]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK: Cost of 0 for VF 2: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
-; CHECK: Cost of 0 for VF 2: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
-; CHECK: Cost of 0 for VF 2: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 2: IR %load1 = load float, ptr %arrayidx, align 4
-; CHECK: Cost of 0 for VF 2: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
-; CHECK: Cost of 0 for VF 2: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
-; CHECK: Cost of 0 for VF 2: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (or) vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11:%[0-9]+]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (or) vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 2: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK: Cost of 0 for VF 2: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 0 for VF 2: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
-; CHECK: Cost of 0 for VF 2: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
-; CHECK: Cost of 1 for VF 4: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 4: exit condition instruction %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 4: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK: Cost of 0 for VF 4: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%load1> = load vp<[[VP5]]>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = not ir<%cmp1>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 0 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: vector loop backedge
-; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 4: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK: Cost of 0 for VF 4: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
-; CHECK: Cost of 0 for VF 4: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
-; CHECK: Cost of 0 for VF 4: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 4: IR %load1 = load float, ptr %arrayidx, align 4
-; CHECK: Cost of 0 for VF 4: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
-; CHECK: Cost of 0 for VF 4: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
-; CHECK: Cost of 0 for VF 4: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 0 for VF 4: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 4: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 4: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK: Cost of 0 for VF 4: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 0 for VF 4: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
-; CHECK: Cost of 0 for VF 4: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
-; CHECK: Cost of 1 for VF 8: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 8: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 8: exit condition instruction %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 8: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 8: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 8: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 8: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK: Cost of 0 for VF 8: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 8: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%load1> = load vp<[[VP5]]>
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
-; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
-; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP7]]> = not ir<%cmp1>
-; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
-; CHECK: Cost of 0 for VF 8: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 0 for VF 8: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 8: vector loop backedge
-; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 8: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK: Cost of 0 for VF 8: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
-; CHECK: Cost of 0 for VF 8: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
-; CHECK: Cost of 0 for VF 8: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 8: IR %load1 = load float, ptr %arrayidx, align 4
-; CHECK: Cost of 0 for VF 8: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
-; CHECK: Cost of 0 for VF 8: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
-; CHECK: Cost of 0 for VF 8: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 0 for VF 8: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 8: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 8: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 8: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 8: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK: Cost of 0 for VF 8: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 0 for VF 8: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
-; CHECK: Cost of 0 for VF 8: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
-; CHECK: Cost of 1 for VF 16: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 16: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 16: exit condition instruction %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 16: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 16: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 16: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 16: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK: Cost of 0 for VF 16: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 16: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%load1> = load vp<[[VP5]]>
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
-; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
-; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP7]]> = not ir<%cmp1>
-; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
-; CHECK: Cost of 0 for VF 16: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 0 for VF 16: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 16: vector loop backedge
-; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 16: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK: Cost of 0 for VF 16: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
-; CHECK: Cost of 0 for VF 16: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
-; CHECK: Cost of 0 for VF 16: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 16: IR %load1 = load float, ptr %arrayidx, align 4
-; CHECK: Cost of 0 for VF 16: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
-; CHECK: Cost of 0 for VF 16: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
-; CHECK: Cost of 0 for VF 16: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 0 for VF 16: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 16: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 16: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 16: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 16: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK: Cost of 0 for VF 16: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 0 for VF 16: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
-; CHECK: Cost of 0 for VF 16: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
-; CHECK: Cost of 1 for VF 32: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 32: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 32: exit condition instruction %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 32: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 32: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 32: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 32: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK: Cost of 0 for VF 32: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 32: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
-; CHECK: Cost of 8 for VF 32: WIDEN ir<%load1> = load vp<[[VP5]]>
-; CHECK: Cost of 8 for VF 32: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
-; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
-; CHECK: Cost of 2 for VF 32: EMIT vp<[[VP7]]> = not ir<%cmp1>
-; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
-; CHECK: Cost of 0 for VF 32: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 0 for VF 32: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 32: vector loop backedge
-; CHECK: Cost of 0 for VF 32: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 32: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 32: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 32: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK: Cost of 0 for VF 32: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
-; CHECK: Cost of 0 for VF 32: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
-; CHECK: Cost of 0 for VF 32: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 32: IR %load1 = load float, ptr %arrayidx, align 4
-; CHECK: Cost of 0 for VF 32: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
-; CHECK: Cost of 0 for VF 32: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
-; CHECK: Cost of 0 for VF 32: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 0 for VF 32: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 32: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 32: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 32: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 32: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 32: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 32: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK: Cost of 0 for VF 32: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 0 for VF 32: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
-; CHECK: Cost of 0 for VF 32: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
-; CHECK: Cost of 1 for VF 64: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 64: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 64: exit condition instruction %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 64: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 64: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 64: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 64: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK: Cost of 0 for VF 64: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 64: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
-; CHECK: Cost of 16 for VF 64: WIDEN ir<%load1> = load vp<[[VP5]]>
-; CHECK: Cost of 16 for VF 64: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
-; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
-; CHECK: Cost of 4 for VF 64: EMIT vp<[[VP7]]> = not ir<%cmp1>
-; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
-; CHECK: Cost of 0 for VF 64: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 0 for VF 64: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 64: vector loop backedge
-; CHECK: Cost of 0 for VF 64: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 64: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 64: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 64: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK: Cost of 0 for VF 64: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
-; CHECK: Cost of 0 for VF 64: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
-; CHECK: Cost of 0 for VF 64: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 64: IR %load1 = load float, ptr %arrayidx, align 4
-; CHECK: Cost of 0 for VF 64: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
-; CHECK: Cost of 0 for VF 64: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
-; CHECK: Cost of 0 for VF 64: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 0 for VF 64: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 64: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 64: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 64: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 64: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 64: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 64: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK: Cost of 0 for VF 64: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 0 for VF 64: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
-; CHECK: Cost of 0 for VF 64: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
-; CHECK: Cost of 1 for VF 128: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 128: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 128: exit condition instruction %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 128: forced scalar %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 128: WIDEN-REDUCTION-PHI ir<%all.off.next> = phi (any-of) ir<false>, vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 128: WIDEN-REDUCTION-PHI ir<%any.0.off09> = phi (any-of) ir<false>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 128: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
-; CHECK: Cost of 0 for VF 128: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 128: vp<[[VP5]]> = vector-pointer inbounds ir<%arrayidx>
-; CHECK: Cost of 32 for VF 128: WIDEN ir<%load1> = load vp<[[VP5]]>
-; CHECK: Cost of 32 for VF 128: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
-; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP6]]> = or ir<%any.0.off09>, ir<%cmp1>
-; CHECK: Cost of 8 for VF 128: EMIT vp<[[VP7]]> = not ir<%cmp1>
-; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP8]]> = or ir<%all.off.next>, vp<[[VP7]]>
-; CHECK: Cost of 0 for VF 128: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 0 for VF 128: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 128: vector loop backedge
-; CHECK: Cost of 0 for VF 128: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 128: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<true>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 128: EMIT-SCALAR vp<%bc.merge.rdx>.1 = phi [ vp<[[VP14]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 128: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK: Cost of 0 for VF 128: IR %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
-; CHECK: Cost of 0 for VF 128: IR %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ] (extra operand: vp<%bc.merge.rdx>.1 from scalar.ph)
-; CHECK: Cost of 0 for VF 128: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK: Cost of 0 for VF 128: IR %load1 = load float, ptr %arrayidx, align 4
-; CHECK: Cost of 0 for VF 128: IR %cmp1 = fcmp olt float %load1, 0.000000e+00
-; CHECK: Cost of 0 for VF 128: IR %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
-; CHECK: Cost of 0 for VF 128: IR %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 0 for VF 128: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 0 for VF 128: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP10]]> = compute-reduction-result (or) vp<[[VP8]]>
-; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP11]]> = freeze vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 128: EMIT vp<[[VP12]]> = not vp<[[VP11]]>
-; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP13]]> = compute-reduction-result (or) vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 128: EMIT vp<[[VP14]]> = freeze vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 128: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 128: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 128: IR %.any.0.off0.lcssa = phi i1 [ %.any.0.off0, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK: Cost of 0 for VF 128: IR %all.off.lcssa = phi i1 [ %all.off, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 0 for VF 128: IR %0 = select i1 %.any.0.off0.lcssa, i32 2, i32 3
-; CHECK: Cost of 0 for VF 128: IR %1 = select i1 %all.off.lcssa, i32 1, i32 %0
-; CHECK: LV: Selecting VF: 16.
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %all.off.next = phi i1 [ true, %entry ], [ %all.off, %loop ]
- %any.0.off09 = phi i1 [ false, %entry ], [ %.any.0.off0, %loop ]
- %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
- %load1 = load float, ptr %arrayidx, align 4
- %cmp1 = fcmp olt float %load1, 0.000000e+00
- %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
- %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
- %iv.next = add nuw nsw i64 %iv, 1
- %ec = icmp eq i64 %iv.next, %n
- br i1 %ec, label %exit, label %loop
-
-exit:
- %0 = select i1 %.any.0.off0, i32 2, i32 3
- %1 = select i1 %all.off, i32 1, i32 %0
- ret i32 %1
-}
More information about the llvm-commits
mailing list