[llvm] [LV][NFC] Regenerate CHECK lines for some tests (PR #194842)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 06:44:25 PDT 2026
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/194842
>From e6345688e7db08397918b38f5a805dcc55aeb919 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Wed, 29 Apr 2026 13:42:47 +0000
Subject: [PATCH] [LV][NFC] Regenerate CHECK lines for some tests
---
.../AArch64/aarch64-predication.ll | 77 +-
.../AArch64/masked-call-scalarize.ll | 231 +-
.../LoopVectorize/AArch64/masked-call.ll | 696 ++--
.../AArch64/sve-vscale-based-trip-counts.ll | 248 +-
.../Transforms/LoopVectorize/ARM/mve-qabs.ll | 265 +-
.../LoopVectorize/ARM/mve-reductions.ll | 1304 +++----
.../LoopVectorize/ARM/mve-saddsatcost.ll | 37 +-
.../LoopVectorize/ARM/mve-selectandorcost.ll | 129 +-
.../LoopVectorize/ARM/pointer_iv.ll | 732 ++--
.../X86/pr55096-scalarize-add.ll | 58 +-
.../x86-interleaved-accesses-masked-group.ll | 3006 +++++++++--------
...86-interleaved-store-accesses-with-gaps.ll | 390 +--
.../LoopVectorize/X86/x86-predication.ll | 582 ++--
.../LoopVectorize/float-induction.ll | 2610 +++++++-------
.../LoopVectorize/if-pred-non-void.ll | 1034 +++---
.../LoopVectorize/if-pred-stores.ll | 1137 ++++---
16 files changed, 6346 insertions(+), 6190 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
index ba8a4c735b94d..4e969fbdf29d8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; REQUIRES: asserts
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefix=COST
; RUN: opt < %s -passes=loop-vectorize,instcombine,simplifycfg -force-vector-interleave=1 -force-vector-width=2 -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
@@ -15,45 +15,46 @@ target triple = "aarch64--linux-gnu"
;
;
define i64 @predicated_udiv_scalarized_operand(ptr %a, i64 %x) {
-; CHECK-LABEL: @predicated_udiv_scalarized_operand(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE2:%.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY]] ], [ [[TMP17:%.*]], [[PRED_UDIV_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i8], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-LABEL: define i64 @predicated_udiv_scalarized_operand(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP16:%.*]], %[[PRED_UDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <2 x i64> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], [[X:%.*]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[TMP6]], [[TMP5]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
-; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.if1:
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP11]], [[X]]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP14:%.*]] = udiv i64 [[TMP13]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP14]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.continue2:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i64> [ [[TMP9]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP15]], [[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i64> [[TMP16]], <2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP17]] = add <2 x i64> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <2 x i64> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
+; CHECK: [[PRED_UDIV_IF]]:
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], [[X]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = udiv i64 [[TMP5]], [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i64 0
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
+; CHECK: [[PRED_UDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
+; CHECK: [[PRED_UDIV_IF1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw i64 [[TMP10]], [[X]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = udiv i64 [[TMP12]], [[TMP11]]
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP13]], i64 1
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
+; CHECK: [[PRED_UDIV_CONTINUE2]]:
+; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i64> [ [[TMP8]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP14]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[TMP15]], <2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP16]] = add <2 x i64> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP18]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: for.end:
-; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[TMP17]])
-; CHECK-NEXT: ret i64 [[TMP19]]
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP17]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[TMP16]])
+; CHECK-NEXT: ret i64 [[TMP18]]
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
index 8d85bbabcceb7..9b6858b0ac545 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=TFNONE
; RUN: opt < %s -passes=loop-vectorize,simplifycfg -force-vector-interleave=1 -tail-folding-policy=must-fold-tail -S | FileCheck %s --check-prefixes=TFCOMMON,TFALWAYS
; RUN: opt < %s -passes=loop-vectorize,simplifycfg -force-vector-interleave=1 -tail-folding-policy=prefer-fold-tail -S | FileCheck %s --check-prefixes=TFCOMMON,TFFALLBACK
@@ -7,141 +7,144 @@
target triple = "aarch64-unknown-linux-gnu"
define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
-; TFNONE-LABEL: @test_widen_exp_v2(
-; TFNONE-NEXT: entry:
-; TFNONE-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
+; TFNONE-LABEL: define void @test_widen_exp_v2(
+; TFNONE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; TFNONE-NEXT: [[ENTRY:.*]]:
+; TFNONE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFNONE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
-; TFNONE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFNONE: vector.ph:
+; TFNONE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; TFNONE: [[VECTOR_PH]]:
; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]]
-; TFNONE: vector.body:
-; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP7:%.*]] = load double, ptr [[P2:%.*]], align 8
-; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
+; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFNONE: [[VECTOR_BODY]]:
+; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFNONE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
+; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 0
; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
; TFNONE-NEXT: [[TMP2:%.*]] = call <2 x double> @exp_fixed(<2 x double> [[BROADCAST_SPLAT]])
; TFNONE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[TMP2]], zeroinitializer
; TFNONE-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
-; TFNONE-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
-; TFNONE-NEXT: store double [[TMP14]], ptr [[P:%.*]], align 8
+; TFNONE-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
+; TFNONE-NEXT: store double [[TMP4]], ptr [[P]], align 8
; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TFNONE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; TFNONE: middle.block:
+; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
-; TFNONE: scalar.ph:
-; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFNONE-NEXT: br label [[LOOP:%.*]]
-; TFNONE: loop:
-; TFNONE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_END:%.*]] ]
+; TFNONE-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
+; TFNONE: [[SCALAR_PH]]:
+; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; TFNONE-NEXT: br label %[[LOOP:.*]]
+; TFNONE: [[LOOP]]:
+; TFNONE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_END:.*]] ]
; TFNONE-NEXT: [[LD:%.*]] = load double, ptr [[P2]], align 8
; TFNONE-NEXT: [[EXP:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR2:[0-9]+]]
; TFNONE-NEXT: [[COND1:%.*]] = fcmp ogt double [[EXP]], 0.000000e+00
-; TFNONE-NEXT: br i1 [[COND1]], label [[LOOP_MIDDLE:%.*]], label [[LOOP_END]]
-; TFNONE: loop.middle:
-; TFNONE-NEXT: br label [[LOOP_END]]
-; TFNONE: loop.end:
-; TFNONE-NEXT: [[SINK:%.*]] = phi double [ 0.000000e+00, [[LOOP_MIDDLE]] ], [ 1.000000e+00, [[LOOP]] ]
+; TFNONE-NEXT: br i1 [[COND1]], label %[[LOOP_MIDDLE:.*]], label %[[LOOP_END]]
+; TFNONE: [[LOOP_MIDDLE]]:
+; TFNONE-NEXT: br label %[[LOOP_END]]
+; TFNONE: [[LOOP_END]]:
+; TFNONE-NEXT: [[SINK:%.*]] = phi double [ 0.000000e+00, %[[LOOP_MIDDLE]] ], [ 1.000000e+00, %[[LOOP]] ]
; TFNONE-NEXT: store double [[SINK]], ptr [[P]], align 8
; TFNONE-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; TFNONE-NEXT: [[COND2:%.*]] = icmp eq i64 [[IV]], [[N]]
-; TFNONE-NEXT: br i1 [[COND2]], label [[END]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; TFNONE: end:
+; TFNONE-NEXT: br i1 [[COND2]], label %[[END]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; TFNONE: [[END]]:
; TFNONE-NEXT: ret void
;
-; TFCOMMON-LABEL: @test_widen_exp_v2(
-; TFCOMMON-NEXT: entry:
-; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0:%.*]], 1
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N_RND_UP]])
-; TFCOMMON-NEXT: br label [[LOOP:%.*]]
-; TFCOMMON: vector.body:
-; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
-; TFCOMMON-NEXT: [[LD:%.*]] = load double, ptr [[P2:%.*]], align 8
-; TFCOMMON-NEXT: [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR3:[0-9]+]]
-; TFCOMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0
-; TFCOMMON-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; TFCOMMON-NEXT: [[TMP9:%.*]] = fcmp ogt <2 x double> [[TMP8]], zeroinitializer
-; TFCOMMON-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP9]], i64 0
-; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
-; TFCOMMON-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
-; TFCOMMON-NEXT: br i1 [[TMP16]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; TFCOMMON: pred.store.if:
-; TFCOMMON-NEXT: [[SINK:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 0
-; TFCOMMON-NEXT: store double [[SINK]], ptr [[P:%.*]], align 8
-; TFCOMMON-NEXT: br label [[PRED_STORE_CONTINUE]]
-; TFCOMMON: pred.store.continue:
-; TFCOMMON-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
-; TFCOMMON-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE6]]
-; TFCOMMON: pred.store.if1:
-; TFCOMMON-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
-; TFCOMMON-NEXT: store double [[TMP19]], ptr [[P]], align 8
-; TFCOMMON-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; TFCOMMON: pred.store.continue2:
+; TFCOMMON-LABEL: define void @test_widen_exp_v2(
+; TFCOMMON-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; TFCOMMON-NEXT: [[ENTRY:.*]]:
+; TFCOMMON-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFCOMMON: [[VECTOR_BODY]]:
+; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; TFCOMMON-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
+; TFCOMMON-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR3:[0-9]+]]
+; TFCOMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
+; TFCOMMON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; TFCOMMON-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
+; TFCOMMON-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
+; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFCOMMON-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TFCOMMON-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TFCOMMON: [[PRED_STORE_IF]]:
+; TFCOMMON-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 0
+; TFCOMMON-NEXT: store double [[TMP6]], ptr [[P]], align 8
+; TFCOMMON-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TFCOMMON: [[PRED_STORE_CONTINUE]]:
+; TFCOMMON-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TFCOMMON-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TFCOMMON: [[PRED_STORE_IF1]]:
+; TFCOMMON-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
+; TFCOMMON-NEXT: store double [[TMP8]], ptr [[P]], align 8
+; TFCOMMON-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TFCOMMON: [[PRED_STORE_CONTINUE2]]:
; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[N_RND_UP]])
-; TFCOMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFCOMMON-NEXT: [[TMP17:%.*]] = xor i1 [[TMP15]], true
-; TFCOMMON-NEXT: br i1 [[TMP17]], label [[END:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
-; TFCOMMON: end:
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFCOMMON-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFCOMMON-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; TFCOMMON-NEXT: br i1 [[TMP10]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFCOMMON: [[END]]:
; TFCOMMON-NEXT: ret void
;
-; TFA_INTERLEAVE-LABEL: @test_widen_exp_v2(
-; TFA_INTERLEAVE-NEXT: entry:
-; TFA_INTERLEAVE-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0:%.*]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N_RND_UP]])
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 2, i64 [[N_RND_UP]])
-; TFA_INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
-; TFA_INTERLEAVE: vector.body:
-; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE9:%.*]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE9]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], [[PRED_STORE_CONTINUE9]] ]
-; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = load double, ptr [[P2:%.*]], align 8
-; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR3:[0-9]+]]
-; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0
-; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = fcmp ogt <2 x double> [[TMP12]], zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP14]], i64 0
-; TFA_INTERLEAVE-NEXT: [[PREDPHI3:%.*]] = select i1 [[TMP7]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
-; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP19]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; TFA_INTERLEAVE: pred.store.if:
-; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <2 x double> [[PREDPHI3]], i64 0
-; TFA_INTERLEAVE-NEXT: store double [[TMP20]], ptr [[P:%.*]], align 8
-; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE]]
-; TFA_INTERLEAVE: pred.store.continue:
-; TFA_INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP29]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
-; TFA_INTERLEAVE: pred.store.if3:
-; TFA_INTERLEAVE-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[PREDPHI3]], i64 1
-; TFA_INTERLEAVE-NEXT: store double [[TMP22]], ptr [[P]], align 8
-; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE5]]
-; TFA_INTERLEAVE: pred.store.continue4:
-; TFA_INTERLEAVE-NEXT: [[TMP31:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i64 0
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP31]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7:%.*]]
-; TFA_INTERLEAVE: pred.store.if5:
-; TFA_INTERLEAVE-NEXT: [[TMP32:%.*]] = extractelement <2 x double> [[PREDPHI3]], i64 0
-; TFA_INTERLEAVE-NEXT: store double [[TMP32]], ptr [[P]], align 8
-; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE7]]
-; TFA_INTERLEAVE: pred.store.continue6:
-; TFA_INTERLEAVE-NEXT: [[TMP25:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i64 1
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP25]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9]]
-; TFA_INTERLEAVE: pred.store.if7:
-; TFA_INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <2 x double> [[PREDPHI3]], i64 1
-; TFA_INTERLEAVE-NEXT: store double [[TMP34]], ptr [[P]], align 8
-; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE9]]
-; TFA_INTERLEAVE: pred.store.continue8:
+; TFA_INTERLEAVE-LABEL: define void @test_widen_exp_v2(
+; TFA_INTERLEAVE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; TFA_INTERLEAVE-NEXT: [[ENTRY:.*]]:
+; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 2, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFA_INTERLEAVE: [[VECTOR_BODY]]:
+; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT9:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR3:[0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
+; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
+; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TFA_INTERLEAVE: [[PRED_STORE_IF]]:
+; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 0
+; TFA_INTERLEAVE-NEXT: store double [[TMP6]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE]]:
+; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; TFA_INTERLEAVE: [[PRED_STORE_IF3]]:
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
+; TFA_INTERLEAVE-NEXT: store double [[TMP8]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]:
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i64 0
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; TFA_INTERLEAVE: [[PRED_STORE_IF5]]:
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 0
+; TFA_INTERLEAVE-NEXT: store double [[TMP10]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE6]]:
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i64 1
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; TFA_INTERLEAVE: [[PRED_STORE_IF7]]:
+; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
+; TFA_INTERLEAVE-NEXT: store double [[TMP12]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE8]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[N_RND_UP]])
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT10]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP13]], i64 [[N_RND_UP]])
-; TFA_INTERLEAVE-NEXT: [[TMP26:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFA_INTERLEAVE-NEXT: [[TMP28:%.*]] = xor i1 [[TMP26]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP28]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; TFA_INTERLEAVE: end:
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT9]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP13]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFA_INTERLEAVE: [[END]]:
; TFA_INTERLEAVE-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 24b199dabfbce..d5026ce689676 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=TFNONE
; RUN: opt < %s -passes=loop-vectorize,simplifycfg -force-vector-interleave=1 -tail-folding-policy=must-fold-tail -S | FileCheck %s --check-prefixes=TFCOMMON,TFALWAYS
; RUN: opt < %s -passes=loop-vectorize,simplifycfg -force-vector-interleave=1 -tail-folding-policy=prefer-fold-tail -S | FileCheck %s --check-prefixes=TFCOMMON,TFFALLBACK
@@ -14,21 +14,21 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFNONE-NEXT: [[ENTRY:.*:]]
; TFNONE-NEXT: br label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFNONE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
-; TFNONE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
-; TFNONE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFNONE-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP2]], align 8
+; TFNONE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; TFNONE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH:.*]]
@@ -50,56 +50,56 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFCOMMON-LABEL: define void @test_widen(
; TFCOMMON-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; TFCOMMON-NEXT: [[ENTRY:.*]]:
-; TFCOMMON-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFCOMMON-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; TFCOMMON-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; TFCOMMON: [[VECTOR_BODY]]:
; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFCOMMON-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFCOMMON-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFCOMMON-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TFCOMMON-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFCOMMON-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFCOMMON-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFCOMMON-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFCOMMON-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
-; TFCOMMON-NEXT: br i1 [[TMP9]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFCOMMON-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFCOMMON-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; TFCOMMON-NEXT: br i1 [[TMP6]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; TFCOMMON: [[FOR_COND_CLEANUP]]:
; TFCOMMON-NEXT: ret void
;
; TFA_INTERLEAVE-LABEL: define void @test_widen(
; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; TFA_INTERLEAVE-NEXT: [[ENTRY:.*]]:
-; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 1
-; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP11]], ptr align 8 [[TMP13]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP5]]
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP21]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; TFA_INTERLEAVE: [[FOR_COND_CLEANUP]]:
; TFA_INTERLEAVE-NEXT: ret void
;
@@ -128,23 +128,23 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFNONE-NEXT: [[ENTRY:.*:]]
; TFNONE-NEXT: br label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFNONE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
-; TFNONE-NEXT: [[TMP7:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 50)
-; TFNONE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i1> [[TMP7]])
-; TFNONE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> zeroinitializer
-; TFNONE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TFNONE-NEXT: store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP10]], align 8
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; TFNONE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; TFNONE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP2]], align 8
+; TFNONE-NEXT: [[TMP3:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 50)
+; TFNONE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i1> [[TMP3]])
+; TFNONE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> zeroinitializer
+; TFNONE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; TFNONE-NEXT: store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP5]], align 8
+; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; TFNONE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH:.*]]
@@ -153,16 +153,16 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFNONE: [[FOR_BODY]]:
; TFNONE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ], [ [[N_VEC]], %[[SCALAR_PH]] ]
; TFNONE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFNONE-NEXT: [[TMP12:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; TFNONE-NEXT: [[CMP:%.*]] = icmp ugt i64 [[TMP12]], 50
+; TFNONE-NEXT: [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; TFNONE-NEXT: [[CMP:%.*]] = icmp ugt i64 [[TMP7]], 50
; TFNONE-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END]]
; TFNONE: [[IF_THEN]]:
-; TFNONE-NEXT: [[TMP9:%.*]] = call i64 @foo(i64 [[TMP12]]) #[[ATTR4]]
+; TFNONE-NEXT: [[TMP8:%.*]] = call i64 @foo(i64 [[TMP7]]) #[[ATTR4]]
; TFNONE-NEXT: br label %[[IF_END]]
; TFNONE: [[IF_END]]:
-; TFNONE-NEXT: [[TMP14:%.*]] = phi i64 [ [[TMP9]], %[[IF_THEN]] ], [ 0, %[[FOR_BODY]] ]
+; TFNONE-NEXT: [[TMP9:%.*]] = phi i64 [ [[TMP8]], %[[IF_THEN]] ], [ 0, %[[FOR_BODY]] ]
; TFNONE-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFNONE-NEXT: store i64 [[TMP14]], ptr [[ARRAYIDX1]], align 8
+; TFNONE-NEXT: store i64 [[TMP9]], ptr [[ARRAYIDX1]], align 8
; TFNONE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; TFNONE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
; TFNONE-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -172,65 +172,65 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFCOMMON-LABEL: define void @test_if_then(
; TFCOMMON-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFCOMMON-NEXT: [[ENTRY:.*]]:
-; TFCOMMON-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFCOMMON-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; TFCOMMON-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; TFCOMMON: [[VECTOR_BODY]]:
; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFCOMMON-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFCOMMON-NEXT: [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
-; TFCOMMON-NEXT: [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
-; TFCOMMON-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP7]])
-; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> zeroinitializer
-; TFCOMMON-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TFCOMMON-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFCOMMON-NEXT: [[TMP3:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
+; TFCOMMON-NEXT: [[TMP4:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP3]], <vscale x 2 x i1> zeroinitializer
+; TFCOMMON-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP4]])
+; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> zeroinitializer
+; TFCOMMON-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFCOMMON-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFCOMMON-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; TFCOMMON-NEXT: br i1 [[TMP11]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; TFCOMMON-NEXT: [[TMP7:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFCOMMON-NEXT: [[TMP8:%.*]] = xor i1 [[TMP7]], true
+; TFCOMMON-NEXT: br i1 [[TMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; TFCOMMON: [[FOR_COND_CLEANUP]]:
; TFCOMMON-NEXT: ret void
;
; TFA_INTERLEAVE-LABEL: define void @test_if_then(
; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFA_INTERLEAVE-NEXT: [[ENTRY:.*]]:
-; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 1
-; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
-; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], splat (i64 50)
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP13]])
-; TFA_INTERLEAVE-NEXT: [[TMP16:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[TMP14]])
-; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP20]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = add i64 [[INDEX_NEXT]], [[TMP5]]
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
+; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], splat (i64 50)
+; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP5]], <vscale x 2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP7]])
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[TMP8]])
+; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP5]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP23]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFA_INTERLEAVE-NEXT: [[TMP25:%.*]] = xor i1 [[TMP24]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP25]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP13]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; TFA_INTERLEAVE: [[FOR_COND_CLEANUP]]:
; TFA_INTERLEAVE-NEXT: ret void
;
@@ -270,25 +270,25 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFNONE-NEXT: [[ENTRY:.*:]]
; TFNONE-NEXT: br label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFNONE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
-; TFNONE-NEXT: [[TMP7:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 50)
-; TFNONE-NEXT: [[TMP8:%.*]] = xor <vscale x 2 x i1> [[TMP7]], splat (i1 true)
-; TFNONE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP8]])
-; TFNONE-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i1> [[TMP7]])
-; TFNONE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP9]]
-; TFNONE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TFNONE-NEXT: store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP11]], align 8
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; TFNONE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; TFNONE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP2]], align 8
+; TFNONE-NEXT: [[TMP3:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 50)
+; TFNONE-NEXT: [[TMP4:%.*]] = xor <vscale x 2 x i1> [[TMP3]], splat (i1 true)
+; TFNONE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP4]])
+; TFNONE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i1> [[TMP3]])
+; TFNONE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP3]], <vscale x 2 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP5]]
+; TFNONE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; TFNONE-NEXT: store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP7]], align 8
+; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; TFNONE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH:.*]]
@@ -297,19 +297,19 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFNONE: [[FOR_BODY]]:
; TFNONE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ], [ [[N_VEC]], %[[SCALAR_PH]] ]
; TFNONE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFNONE-NEXT: [[TMP13:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; TFNONE-NEXT: [[CMP:%.*]] = icmp ugt i64 [[TMP13]], 50
+; TFNONE-NEXT: [[TMP9:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; TFNONE-NEXT: [[CMP:%.*]] = icmp ugt i64 [[TMP9]], 50
; TFNONE-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
; TFNONE: [[IF_THEN]]:
-; TFNONE-NEXT: [[TMP14:%.*]] = call i64 @foo(i64 [[TMP13]]) #[[ATTR5:[0-9]+]]
+; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @foo(i64 [[TMP9]]) #[[ATTR5:[0-9]+]]
; TFNONE-NEXT: br label %[[IF_END]]
; TFNONE: [[IF_ELSE]]:
-; TFNONE-NEXT: [[TMP15:%.*]] = call i64 @foo(i64 0) #[[ATTR5]]
+; TFNONE-NEXT: [[TMP11:%.*]] = call i64 @foo(i64 0) #[[ATTR5]]
; TFNONE-NEXT: br label %[[IF_END]]
; TFNONE: [[IF_END]]:
-; TFNONE-NEXT: [[TMP16:%.*]] = phi i64 [ [[TMP14]], %[[IF_THEN]] ], [ [[TMP15]], %[[IF_ELSE]] ]
+; TFNONE-NEXT: [[TMP12:%.*]] = phi i64 [ [[TMP10]], %[[IF_THEN]] ], [ [[TMP11]], %[[IF_ELSE]] ]
; TFNONE-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFNONE-NEXT: store i64 [[TMP16]], ptr [[ARRAYIDX1]], align 8
+; TFNONE-NEXT: store i64 [[TMP12]], ptr [[ARRAYIDX1]], align 8
; TFNONE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; TFNONE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
; TFNONE-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -319,74 +319,74 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFCOMMON-LABEL: define void @test_widen_if_then_else(
; TFCOMMON-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFCOMMON-NEXT: [[ENTRY:.*]]:
-; TFCOMMON-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFCOMMON-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; TFCOMMON-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; TFCOMMON: [[VECTOR_BODY]]:
; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFCOMMON-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFCOMMON-NEXT: [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
-; TFCOMMON-NEXT: [[TMP7:%.*]] = xor <vscale x 2 x i1> [[TMP6]], splat (i1 true)
-; TFCOMMON-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
-; TFCOMMON-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP8]])
-; TFCOMMON-NEXT: [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
-; TFCOMMON-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP10]])
-; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP9]]
-; TFCOMMON-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TFCOMMON-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFCOMMON-NEXT: [[TMP3:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
+; TFCOMMON-NEXT: [[TMP4:%.*]] = xor <vscale x 2 x i1> [[TMP3]], splat (i1 true)
+; TFCOMMON-NEXT: [[TMP5:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP4]], <vscale x 2 x i1> zeroinitializer
+; TFCOMMON-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP5]])
+; TFCOMMON-NEXT: [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP3]], <vscale x 2 x i1> zeroinitializer
+; TFCOMMON-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP7]])
+; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP3]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[TMP6]]
+; TFCOMMON-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFCOMMON-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFCOMMON-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
-; TFCOMMON-NEXT: br i1 [[TMP14]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; TFCOMMON-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFCOMMON-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; TFCOMMON-NEXT: br i1 [[TMP11]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; TFCOMMON: [[FOR_COND_CLEANUP]]:
; TFCOMMON-NEXT: ret void
;
; TFA_INTERLEAVE-LABEL: define void @test_widen_if_then_else(
; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFA_INTERLEAVE-NEXT: [[ENTRY:.*]]:
-; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 1
-; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
-; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], splat (i64 50)
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = xor <vscale x 2 x i1> [[TMP11]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = xor <vscale x 2 x i1> [[TMP12]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP13]], <vscale x 2 x i1> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP16:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP15]])
-; TFA_INTERLEAVE-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP16]])
-; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP19]])
-; TFA_INTERLEAVE-NEXT: [[TMP22:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[TMP20]])
-; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i64> [[TMP21]], <vscale x 2 x i64> [[TMP17]]
-; TFA_INTERLEAVE-NEXT: [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i64> [[TMP22]], <vscale x 2 x i64> [[TMP18]]
-; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP23]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP23]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP26]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT: [[TMP29:%.*]] = add i64 [[INDEX_NEXT]], [[TMP5]]
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
+; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], splat (i64 50)
+; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = xor <vscale x 2 x i1> [[TMP5]], splat (i1 true)
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = xor <vscale x 2 x i1> [[TMP6]], splat (i1 true)
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP9]])
+; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP10]])
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP5]], <vscale x 2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP13]])
+; TFA_INTERLEAVE-NEXT: [[TMP16:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[TMP14]])
+; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP5]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP11]]
+; TFA_INTERLEAVE-NEXT: [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> [[TMP12]]
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP29]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP30:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFA_INTERLEAVE-NEXT: [[TMP31:%.*]] = xor i1 [[TMP30]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP31]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP21]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; TFA_INTERLEAVE: [[FOR_COND_CLEANUP]]:
; TFA_INTERLEAVE-NEXT: ret void
;
@@ -429,21 +429,21 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
; TFNONE-NEXT: [[ENTRY:.*:]]
; TFNONE-NEXT: br label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFNONE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
-; TFNONE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
-; TFNONE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; TFNONE-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP2]], align 8
+; TFNONE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
+; TFNONE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH:.*]]
@@ -482,21 +482,21 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
; TFFALLBACK-LABEL: define void @test_widen_nomask(
; TFFALLBACK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFFALLBACK-NEXT: [[ENTRY:.*]]:
-; TFFALLBACK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFFALLBACK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFFALLBACK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
; TFFALLBACK-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; TFFALLBACK-NEXT: br label %[[VECTOR_BODY:.*]]
; TFFALLBACK: [[VECTOR_BODY]]:
; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFFALLBACK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
-; TFFALLBACK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
-; TFFALLBACK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFFALLBACK-NEXT: store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; TFFALLBACK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFFALLBACK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; TFFALLBACK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFFALLBACK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP2]], align 8
+; TFFALLBACK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
+; TFFALLBACK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFFALLBACK-NEXT: store <vscale x 2 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; TFFALLBACK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFFALLBACK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; TFFALLBACK: [[MIDDLE_BLOCK]]:
; TFFALLBACK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
; TFFALLBACK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY:.*]]
@@ -557,21 +557,21 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFNONE-NEXT: [[ENTRY:.*:]]
; TFNONE-NEXT: br label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFNONE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
-; TFNONE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
-; TFNONE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; TFNONE-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP2]], align 8
+; TFNONE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
+; TFNONE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH:.*]]
@@ -593,79 +593,79 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFALWAYS-LABEL: define void @test_widen_optmask(
; TFALWAYS-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFALWAYS-NEXT: [[ENTRY:.*]]:
-; TFALWAYS-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; TFALWAYS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFALWAYS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFALWAYS-NEXT: br label %[[VECTOR_BODY:.*]]
; TFALWAYS: [[VECTOR_BODY]]:
; TFALWAYS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFALWAYS-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFALWAYS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFALWAYS-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFALWAYS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TFALWAYS-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFALWAYS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFALWAYS-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFALWAYS-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFALWAYS-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFALWAYS-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
-; TFALWAYS-NEXT: br i1 [[TMP9]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; TFALWAYS-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFALWAYS-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; TFALWAYS-NEXT: br i1 [[TMP6]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; TFALWAYS: [[FOR_COND_CLEANUP]]:
; TFALWAYS-NEXT: ret void
;
; TFFALLBACK-LABEL: define void @test_widen_optmask(
; TFFALLBACK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFFALLBACK-NEXT: [[ENTRY:.*]]:
-; TFFALLBACK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; TFFALLBACK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFFALLBACK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFFALLBACK-NEXT: br label %[[VECTOR_BODY:.*]]
; TFFALLBACK: [[VECTOR_BODY]]:
; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFFALLBACK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFFALLBACK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFFALLBACK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TFFALLBACK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFFALLBACK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFFALLBACK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFFALLBACK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFFALLBACK-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFFALLBACK-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
-; TFFALLBACK-NEXT: br i1 [[TMP9]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; TFFALLBACK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFFALLBACK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; TFFALLBACK-NEXT: br i1 [[TMP6]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; TFFALLBACK: [[FOR_COND_CLEANUP]]:
; TFFALLBACK-NEXT: ret void
;
; TFA_INTERLEAVE-LABEL: define void @test_widen_optmask(
; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
; TFA_INTERLEAVE-NEXT: [[ENTRY:.*]]:
-; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 1
-; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
-; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP11]], ptr align 8 [[TMP13]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP5]]
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
+; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP21]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; TFA_INTERLEAVE: [[FOR_COND_CLEANUP]]:
; TFA_INTERLEAVE-NEXT: ret void
;
@@ -696,27 +696,27 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFNONE-NEXT: [[ENTRY:.*:]]
; TFNONE-NEXT: br label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFNONE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
-; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP6]], align 8
-; TFNONE-NEXT: [[TMP7:%.*]] = fmul <vscale x 2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; TFNONE-NEXT: [[TMP8:%.*]] = fptoui <vscale x 2 x double> [[WIDE_LOAD]] to <vscale x 2 x i64>
-; TFNONE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP8]], <vscale x 2 x i1> splat (i1 true))
-; TFNONE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP9]], ptr [[TMP10]], align 8
-; TFNONE-NEXT: [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP7]])
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; TFNONE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; TFNONE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; TFNONE-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
+; TFNONE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP2]], align 8
+; TFNONE-NEXT: [[TMP3:%.*]] = fmul <vscale x 2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; TFNONE-NEXT: [[TMP4:%.*]] = fptoui <vscale x 2 x double> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; TFNONE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP4]], <vscale x 2 x i1> splat (i1 true))
+; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFNONE-NEXT: store <vscale x 2 x i64> [[TMP5]], ptr [[TMP6]], align 8
+; TFNONE-NEXT: [[TMP7]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP3]])
+; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; TFNONE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH:.*]]
@@ -724,7 +724,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFNONE-NEXT: br label %[[FOR_BODY:.*]]
; TFNONE: [[FOR_BODY]]:
; TFNONE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[N_VEC]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; TFNONE-NEXT: [[FMA_SUM:%.*]] = phi double [ [[TMP11]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
+; TFNONE-NEXT: [[FMA_SUM:%.*]] = phi double [ [[TMP7]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
; TFNONE-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[B]], i64 [[INDVARS_IV]]
; TFNONE-NEXT: [[LOAD:%.*]] = load double, ptr [[GEP]], align 8
; TFNONE-NEXT: [[MULADD]] = tail call double @llvm.fmuladd.f64(double [[LOAD]], double [[M]], double [[FMA_SUM]])
@@ -736,14 +736,14 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFNONE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
; TFNONE-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; TFNONE: [[FOR_COND_CLEANUP]]:
-; TFNONE-NEXT: [[MULADD_LCSSA:%.*]] = phi double [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; TFNONE-NEXT: [[MULADD_LCSSA:%.*]] = phi double [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; TFNONE-NEXT: ret double [[MULADD_LCSSA]]
;
; TFALWAYS-LABEL: define double @test_widen_fmuladd_and_call(
; TFALWAYS-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], double [[M:%.*]]) #[[ATTR0]] {
; TFALWAYS-NEXT: [[ENTRY:.*]]:
-; TFALWAYS-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; TFALWAYS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFALWAYS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFALWAYS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFALWAYS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
@@ -751,29 +751,29 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFALWAYS: [[VECTOR_BODY]]:
; TFALWAYS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFALWAYS-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; TFALWAYS-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
-; TFALWAYS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; TFALWAYS-NEXT: [[TMP6:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
-; TFALWAYS-NEXT: [[TMP7:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
-; TFALWAYS-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFALWAYS-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP8]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFALWAYS-NEXT: [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> splat (double -0.000000e+00)
-; TFALWAYS-NEXT: [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP10]])
-; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TFALWAYS-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; TFALWAYS-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
+; TFALWAYS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; TFALWAYS-NEXT: [[TMP3:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
+; TFALWAYS-NEXT: [[TMP4:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
+; TFALWAYS-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFALWAYS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFALWAYS-NEXT: [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> splat (double -0.000000e+00)
+; TFALWAYS-NEXT: [[TMP8]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP7]])
+; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFALWAYS-NEXT: [[TMP12:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFALWAYS-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
-; TFALWAYS-NEXT: br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; TFALWAYS-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFALWAYS-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; TFALWAYS-NEXT: br i1 [[TMP10]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; TFALWAYS: [[FOR_COND_CLEANUP]]:
-; TFALWAYS-NEXT: ret double [[TMP11]]
+; TFALWAYS-NEXT: ret double [[TMP8]]
;
; TFFALLBACK-LABEL: define double @test_widen_fmuladd_and_call(
; TFFALLBACK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], double [[M:%.*]]) #[[ATTR0]] {
; TFFALLBACK-NEXT: [[ENTRY:.*]]:
-; TFFALLBACK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; TFFALLBACK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFFALLBACK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFFALLBACK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFFALLBACK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
@@ -781,32 +781,32 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFFALLBACK: [[VECTOR_BODY]]:
; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
-; TFFALLBACK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; TFFALLBACK-NEXT: [[TMP6:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
-; TFFALLBACK-NEXT: [[TMP7:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
-; TFFALLBACK-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFFALLBACK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP8]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFFALLBACK-NEXT: [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> splat (double -0.000000e+00)
-; TFFALLBACK-NEXT: [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP10]])
-; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TFFALLBACK-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
+; TFFALLBACK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; TFFALLBACK-NEXT: [[TMP3:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
+; TFFALLBACK-NEXT: [[TMP4:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
+; TFFALLBACK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFFALLBACK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFFALLBACK-NEXT: [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> splat (double -0.000000e+00)
+; TFFALLBACK-NEXT: [[TMP8]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP7]])
+; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFFALLBACK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFFALLBACK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
-; TFFALLBACK-NEXT: br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; TFFALLBACK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFFALLBACK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; TFFALLBACK-NEXT: br i1 [[TMP10]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; TFFALLBACK: [[FOR_COND_CLEANUP]]:
-; TFFALLBACK-NEXT: ret double [[TMP11]]
+; TFFALLBACK-NEXT: ret double [[TMP8]]
;
; TFA_INTERLEAVE-LABEL: define double @test_widen_fmuladd_and_call(
; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], double [[M:%.*]]) #[[ATTR0]] {
; TFA_INTERLEAVE-NEXT: [[ENTRY:.*]]:
-; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 1
-; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -814,34 +814,34 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr double, ptr [[TMP7]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
-; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
-; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD3]], [[BROADCAST_SPLAT]]
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
-; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD3]] to <vscale x 2 x i64>
-; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP13]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: [[TMP16:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP5]]
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP16]], ptr align 8 [[TMP20]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP11]], <vscale x 2 x double> splat (double -0.000000e+00)
-; TFA_INTERLEAVE-NEXT: [[TMP22:%.*]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP21]])
-; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> [[TMP12]], <vscale x 2 x double> splat (double -0.000000e+00)
-; TFA_INTERLEAVE-NEXT: [[TMP24]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP22]], <vscale x 2 x double> [[TMP23]])
-; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT: [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP5]]
+; TFA_INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
+; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD3]], [[BROADCAST_SPLAT]]
+; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD3]] to <vscale x 2 x i64>
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i64 [[TMP1]]
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> splat (double -0.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP13]])
+; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> splat (double -0.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[TMP16]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP14]], <vscale x 2 x double> [[TMP15]])
+; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP27]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP28:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; TFA_INTERLEAVE-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP29]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP17]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP19]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; TFA_INTERLEAVE: [[FOR_COND_CLEANUP]]:
-; TFA_INTERLEAVE-NEXT: ret double [[TMP24]]
+; TFA_INTERLEAVE-NEXT: ret double [[TMP16]]
;
entry:
br label %for.body
@@ -878,17 +878,17 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP7:%.*]] = load double, ptr [[P2]], align 8
-; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
+; TFNONE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
+; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 0
; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
; TFNONE-NEXT: [[TMP2:%.*]] = call <2 x double> @exp_fixed(<2 x double> [[BROADCAST_SPLAT]])
; TFNONE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[TMP2]], zeroinitializer
; TFNONE-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
-; TFNONE-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
-; TFNONE-NEXT: store double [[TMP14]], ptr [[P]], align 8
+; TFNONE-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
+; TFNONE-NEXT: store double [[TMP4]], ptr [[P]], align 8
; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TFNONE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
@@ -937,25 +937,25 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
-; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[TMP9:.*]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[TMP9]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[TMP9]] ]
-; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = load double, ptr [[P2]], align 8
-; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7:[0-9]+]]
-; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = fcmp ogt double [[TMP6]], 0.000000e+00
-; TFA_INTERLEAVE-NEXT: [[PREDPHI3:%.*]] = select i1 [[TMP8]], double 0.000000e+00, double 1.000000e+00
-; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = or i1 [[ACTIVE_LANE_MASK]], [[ACTIVE_LANE_MASK2]]
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP14]], label %[[BB8:.*]], label %[[TMP9]]
-; TFA_INTERLEAVE: [[BB8]]:
-; TFA_INTERLEAVE-NEXT: store double [[PREDPHI3]], ptr [[P]], align 8
-; TFA_INTERLEAVE-NEXT: br label %[[TMP9]]
-; TFA_INTERLEAVE: [[TMP9]]:
+; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[TMP6:.*]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[TMP6]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[TMP6]] ]
+; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR7:[0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = fcmp ogt double [[TMP2]], 0.000000e+00
+; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP3]], double 0.000000e+00, double 1.000000e+00
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = or i1 [[ACTIVE_LANE_MASK]], [[ACTIVE_LANE_MASK2]]
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP4]], label %[[BB5:.*]], label %[[TMP6]]
+; TFA_INTERLEAVE: [[BB5]]:
+; TFA_INTERLEAVE-NEXT: store double [[PREDPHI]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: br label %[[TMP6]]
+; TFA_INTERLEAVE: [[TMP6]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = add i64 [[INDEX_NEXT]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP7]], [[TMP0]]
-; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP21]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP8]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; TFA_INTERLEAVE: [[END]]:
; TFA_INTERLEAVE-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
index 2e07b27bb6823..37d9c82920271 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=loop-vectorize,simplifycfg -S < %s | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
@@ -9,8 +9,8 @@ define void @vscale_mul_4(ptr noalias noundef readonly captures(none) %a, ptr no
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP10]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[A]], align 4
@@ -24,10 +24,10 @@ define void @vscale_mul_4(ptr noalias noundef readonly captures(none) %a, ptr no
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[ENTRY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT: [[MUL4:%.*]] = fmul float [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT: [[MUL4:%.*]] = fmul float [[TMP5]], [[TMP6]]
; CHECK-NEXT: store float [[MUL4]], ptr [[ARRAYIDX3]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[TMP1]]
@@ -60,21 +60,21 @@ define void @vscale_mul_8(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MUL1:%.*]] = shl nuw nsw i64 [[TMP0]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP3]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[B]], align 4
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP17]], ptr [[B]], align 4
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP18]], ptr [[TMP16]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP7:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP6]], ptr [[B]], align 4
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP7]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[MUL1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP]]:
@@ -82,10 +82,10 @@ define void @vscale_mul_8(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[ENTRY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP23]], [[TMP24]]
+; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP8]], [[TMP9]]
; CHECK-NEXT: store float [[MUL5]], ptr [[ARRAYIDX4]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[MUL1]]
@@ -118,22 +118,22 @@ define void @vscale_mul_12(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 12
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP25]], ptr [[TMP12]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP5]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[MUL1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY:.*]]
@@ -142,10 +142,10 @@ define void @vscale_mul_12(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP14:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP7]], [[TMP8]]
; CHECK-NEXT: store float [[MUL5]], ptr [[ARRAYIDX4]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[MUL1]]
@@ -178,29 +178,29 @@ define void @vscale_mul_31(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 31
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP3]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP2]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP2]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP17]], ptr [[TMP12]], align 4
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP18]], ptr [[TMP16]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP9:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP8]], ptr [[TMP6]], align 4
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP9]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[MUL1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY:.*]]
@@ -209,10 +209,10 @@ define void @vscale_mul_31(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP23]], [[TMP24]]
+; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP11]], [[TMP12]]
; CHECK-NEXT: store float [[MUL5]], ptr [[ARRAYIDX4]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[MUL1]]
@@ -245,29 +245,29 @@ define void @vscale_mul_64(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 64
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP3]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP2]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP2]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP17]], ptr [[TMP12]], align 4
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP18]], ptr [[TMP16]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP9:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP8]], ptr [[TMP6]], align 4
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP9]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[MUL1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY:.*]]
@@ -276,10 +276,10 @@ define void @vscale_mul_64(ptr noalias noundef readonly captures(none) %a, ptr n
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP23]], [[TMP24]]
+; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT: [[MUL5:%.*]] = fmul float [[TMP11]], [[TMP12]]
; CHECK-NEXT: store float [[MUL5]], ptr [[ARRAYIDX4]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[MUL1]]
@@ -314,33 +314,33 @@ define void @trip_count_with_overflow(ptr noalias noundef readonly captures(none
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP4]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP6]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP6]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP6]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP19:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP18]], ptr [[TMP13]], align 4
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP19]], ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP4]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP4]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP11:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP10]], ptr [[TMP8]], align 4
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP11]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
@@ -352,10 +352,10 @@ define void @trip_count_with_overflow(ptr noalias noundef readonly captures(none
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP25:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT: [[MUL4:%.*]] = fmul float [[TMP24]], [[TMP25]]
+; CHECK-NEXT: [[TMP14:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT: [[MUL4:%.*]] = fmul float [[TMP13]], [[TMP14]]
; CHECK-NEXT: store float [[MUL4]], ptr [[ARRAYIDX3]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[TMP1]]
@@ -389,29 +389,29 @@ define void @trip_count_too_big_for_element_count(ptr noalias noundef readonly c
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 32
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 1
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP4]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP3]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP3]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP19:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP18]], ptr [[TMP13]], align 4
-; CHECK-NEXT: store <vscale x 4 x float> [[TMP19]], ptr [[TMP17]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP3]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP10:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP9]], ptr [[TMP7]], align 4
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP10]], ptr [[TMP8]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY:.*]]
@@ -420,10 +420,10 @@ define void @trip_count_too_big_for_element_count(ptr noalias noundef readonly c
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP25:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT: [[MUL4:%.*]] = fmul float [[TMP24]], [[TMP25]]
+; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT: [[MUL4:%.*]] = fmul float [[TMP12]], [[TMP13]]
; CHECK-NEXT: store float [[MUL4]], ptr [[ARRAYIDX3]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll
index 8a79ea62361a4..fa59cb6746021 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll
@@ -1,67 +1,68 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg -simplifycfg-require-and-preserve-domtree=1 < %s -S -o - | FileCheck %s
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-arm-none-eabi"
define void @arm_abs_q7(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32 %blockSize) #0 {
-; CHECK-LABEL: @arm_abs_q7(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[PSRC2:%.*]] = ptrtoaddr ptr [[PSRC:%.*]] to i32
-; CHECK-NEXT: [[PDST1:%.*]] = ptrtoaddr ptr [[PDST:%.*]] to i32
-; CHECK-NEXT: [[CMP_NOT19:%.*]] = icmp eq i32 [[BLOCKSIZE:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT19]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
-; CHECK: while.body.preheader:
+; CHECK-LABEL: define void @arm_abs_q7(
+; CHECK-SAME: ptr readonly captures(none) [[PSRC:%.*]], ptr captures(none) [[PDST:%.*]], i32 [[BLOCKSIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PSRC2:%.*]] = ptrtoaddr ptr [[PSRC]] to i32
+; CHECK-NEXT: [[PDST1:%.*]] = ptrtoaddr ptr [[PDST]] to i32
+; CHECK-NEXT: [[CMP_NOT19:%.*]] = icmp eq i32 [[BLOCKSIZE]], 0
+; CHECK-NEXT: br i1 [[CMP_NOT19]], label %[[WHILE_END:.*]], label %[[WHILE_BODY_PREHEADER:.*]]
+; CHECK: [[WHILE_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[BLOCKSIZE]], 16
; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[PDST1]], [[PSRC2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP0]], 16
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[MIN_ITERS_CHECK]], i1 true, i1 [[DIFF_CHECK]]
-; CHECK-NEXT: br i1 [[OR_COND]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[BLOCKSIZE]], -16
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[N_VEC]]
-; CHECK-NEXT: [[IND_END3:%.*]] = and i32 [[BLOCKSIZE]], 15
-; CHECK-NEXT: [[IND_END5:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[N_VEC]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[N_VEC]]
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[BLOCKSIZE]], 15
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[INDEX]]
-; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], splat (i8 -128)
-; CHECK-NEXT: [[TMP3:%.*]] = sub <16 x i8> zeroinitializer, [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP4:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> splat (i8 127), <16 x i8> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <16 x i1> [[TMP1]], <16 x i8> [[WIDE_LOAD]], <16 x i8> [[TMP4]]
-; CHECK-NEXT: store <16 x i8> [[TMP5]], ptr [[NEXT_GEP7]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], splat (i8 -128)
+; CHECK-NEXT: [[TMP6:%.*]] = sub <16 x i8> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <16 x i1> [[TMP5]], <16 x i8> splat (i8 127), <16 x i8> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <16 x i1> [[TMP4]], <16 x i8> [[WIDE_LOAD]], <16 x i8> [[TMP7]]
+; CHECK-NEXT: store <16 x i8> [[TMP8]], ptr [[NEXT_GEP3]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[BLOCKSIZE]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[WHILE_END]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[PSRC]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ [[PDST]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
-; CHECK: while.body:
-; CHECK-NEXT: [[PSRC_ADDR_022:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[BLKCNT_021:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[PDST_ADDR_020:%.*]] = phi ptr [ [[INCDEC_PTR13:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[WHILE_END]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[PSRC]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[PDST]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[PSRC_ADDR_022:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BLKCNT_021:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[PDST_ADDR_020:%.*]] = phi ptr [ [[INCDEC_PTR13:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_022]], i32 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[PSRC_ADDR_022]], align 1
-; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i8 [[TMP7]], 0
-; CHECK-NEXT: [[CMP5:%.*]] = icmp eq i8 [[TMP7]], -128
-; CHECK-NEXT: [[SUB:%.*]] = sub i8 0, [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[PSRC_ADDR_022]], align 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i8 [[TMP10]], 0
+; CHECK-NEXT: [[CMP5:%.*]] = icmp eq i8 [[TMP10]], -128
+; CHECK-NEXT: [[SUB:%.*]] = sub i8 0, [[TMP10]]
; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP5]], i8 127, i8 [[SUB]]
-; CHECK-NEXT: [[COND11:%.*]] = select i1 [[CMP1]], i8 [[TMP7]], i8 [[COND]]
+; CHECK-NEXT: [[COND11:%.*]] = select i1 [[CMP1]], i8 [[TMP10]], i8 [[COND]]
; CHECK-NEXT: [[INCDEC_PTR13]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_020]], i32 1
; CHECK-NEXT: store i8 [[COND11]], ptr [[PDST_ADDR_020]], align 1
; CHECK-NEXT: [[DEC]] = add i32 [[BLKCNT_021]], -1
; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: while.end:
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END]], label %[[WHILE_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[WHILE_END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -96,67 +97,68 @@ while.end:
}
define void @arm_abs_q15(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32 %blockSize) #0 {
-; CHECK-LABEL: @arm_abs_q15(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[PSRC2:%.*]] = ptrtoaddr ptr [[PSRC:%.*]] to i32
-; CHECK-NEXT: [[PDST1:%.*]] = ptrtoaddr ptr [[PDST:%.*]] to i32
-; CHECK-NEXT: [[CMP_NOT20:%.*]] = icmp eq i32 [[BLOCKSIZE:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT20]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
-; CHECK: while.body.preheader:
+; CHECK-LABEL: define void @arm_abs_q15(
+; CHECK-SAME: ptr readonly captures(none) [[PSRC:%.*]], ptr captures(none) [[PDST:%.*]], i32 [[BLOCKSIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PSRC2:%.*]] = ptrtoaddr ptr [[PSRC]] to i32
+; CHECK-NEXT: [[PDST1:%.*]] = ptrtoaddr ptr [[PDST]] to i32
+; CHECK-NEXT: [[CMP_NOT20:%.*]] = icmp eq i32 [[BLOCKSIZE]], 0
+; CHECK-NEXT: br i1 [[CMP_NOT20]], label %[[WHILE_END:.*]], label %[[WHILE_BODY_PREHEADER:.*]]
+; CHECK: [[WHILE_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[BLOCKSIZE]], 8
; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[PDST1]], [[PSRC2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP0]], 16
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[MIN_ITERS_CHECK]], i1 true, i1 [[DIFF_CHECK]]
-; CHECK-NEXT: br i1 [[OR_COND]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[BLOCKSIZE]], -8
; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[N_VEC]], 1
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[TMP1]]
-; CHECK-NEXT: [[IND_END3:%.*]] = and i32 [[BLOCKSIZE]], 7
-; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[N_VEC]], 1
-; CHECK-NEXT: [[IND_END5:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[TMP2]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[BLOCKSIZE]], 7
+; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[TMP4]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX7:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX7]]
+; CHECK-NEXT: [[OFFSET_IDX3:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2
-; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <8 x i16> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <8 x i16> [[WIDE_LOAD]], splat (i16 -32768)
-; CHECK-NEXT: [[TMP5:%.*]] = sub <8 x i16> zeroinitializer, [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP6:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> splat (i16 32767), <8 x i16> [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <8 x i1> [[TMP3]], <8 x i16> [[WIDE_LOAD]], <8 x i16> [[TMP6]]
-; CHECK-NEXT: store <8 x i16> [[TMP7]], ptr [[NEXT_GEP8]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <8 x i16> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <8 x i16> [[WIDE_LOAD]], splat (i16 -32768)
+; CHECK-NEXT: [[TMP8:%.*]] = sub <8 x i16> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP9:%.*]] = select <8 x i1> [[TMP7]], <8 x i16> splat (i16 32767), <8 x i16> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[WIDE_LOAD]], <8 x i16> [[TMP9]]
+; CHECK-NEXT: store <8 x i16> [[TMP10]], ptr [[NEXT_GEP4]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[BLOCKSIZE]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[WHILE_END]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[PSRC]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ [[PDST]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
-; CHECK: while.body:
-; CHECK-NEXT: [[PSRC_ADDR_023:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[BLKCNT_022:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[PDST_ADDR_021:%.*]] = phi ptr [ [[INCDEC_PTR13:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[WHILE_END]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PSRC]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[PDST]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[PSRC_ADDR_023:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BLKCNT_022:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[PDST_ADDR_021:%.*]] = phi ptr [ [[INCDEC_PTR13:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_023]], i32 2
-; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[PSRC_ADDR_023]], align 2
-; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i16 [[TMP9]], 0
-; CHECK-NEXT: [[CMP5:%.*]] = icmp eq i16 [[TMP9]], -32768
-; CHECK-NEXT: [[SUB:%.*]] = sub i16 0, [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[PSRC_ADDR_023]], align 2
+; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i16 [[TMP12]], 0
+; CHECK-NEXT: [[CMP5:%.*]] = icmp eq i16 [[TMP12]], -32768
+; CHECK-NEXT: [[SUB:%.*]] = sub i16 0, [[TMP12]]
; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP5]], i16 32767, i16 [[SUB]]
-; CHECK-NEXT: [[COND11:%.*]] = select i1 [[CMP1]], i16 [[TMP9]], i16 [[COND]]
+; CHECK-NEXT: [[COND11:%.*]] = select i1 [[CMP1]], i16 [[TMP12]], i16 [[COND]]
; CHECK-NEXT: [[INCDEC_PTR13]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_021]], i32 2
; CHECK-NEXT: store i16 [[COND11]], ptr [[PDST_ADDR_021]], align 2
; CHECK-NEXT: [[DEC]] = add i32 [[BLKCNT_022]], -1
; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: while.end:
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END]], label %[[WHILE_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[WHILE_END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -191,67 +193,68 @@ while.end:
}
define void @arm_abs_q31(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32 %blockSize) #0 {
-; CHECK-LABEL: @arm_abs_q31(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[PSRC2:%.*]] = ptrtoaddr ptr [[PSRC:%.*]] to i32
-; CHECK-NEXT: [[PDST1:%.*]] = ptrtoaddr ptr [[PDST:%.*]] to i32
-; CHECK-NEXT: [[CMP_NOT14:%.*]] = icmp eq i32 [[BLOCKSIZE:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT14]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
-; CHECK: while.body.preheader:
+; CHECK-LABEL: define void @arm_abs_q31(
+; CHECK-SAME: ptr readonly captures(none) [[PSRC:%.*]], ptr captures(none) [[PDST:%.*]], i32 [[BLOCKSIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PSRC2:%.*]] = ptrtoaddr ptr [[PSRC]] to i32
+; CHECK-NEXT: [[PDST1:%.*]] = ptrtoaddr ptr [[PDST]] to i32
+; CHECK-NEXT: [[CMP_NOT14:%.*]] = icmp eq i32 [[BLOCKSIZE]], 0
+; CHECK-NEXT: br i1 [[CMP_NOT14]], label %[[WHILE_END:.*]], label %[[WHILE_BODY_PREHEADER:.*]]
+; CHECK: [[WHILE_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[BLOCKSIZE]], 4
; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[PDST1]], [[PSRC2]]
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP0]], 16
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[MIN_ITERS_CHECK]], i1 true, i1 [[DIFF_CHECK]]
-; CHECK-NEXT: br i1 [[OR_COND]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[BLOCKSIZE]], -4
; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[N_VEC]], 2
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[TMP1]]
-; CHECK-NEXT: [[IND_END3:%.*]] = and i32 [[BLOCKSIZE]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[N_VEC]], 2
-; CHECK-NEXT: [[IND_END5:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[TMP2]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[BLOCKSIZE]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[TMP4]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX7:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX7]]
+; CHECK-NEXT: [[OFFSET_IDX3:%.*]] = shl i32 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 -2147483648)
-; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <4 x i32> zeroinitializer, [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> splat (i32 2147483647), <4 x i32> [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[TMP6]]
-; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[NEXT_GEP8]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 -2147483648)
+; CHECK-NEXT: [[TMP8:%.*]] = sub nsw <4 x i32> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> splat (i32 2147483647), <4 x i32> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[TMP9]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[NEXT_GEP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[BLOCKSIZE]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[WHILE_END]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[PSRC]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ [[PDST]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
-; CHECK: while.body:
-; CHECK-NEXT: [[PSRC_ADDR_017:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[BLKCNT_016:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[PDST_ADDR_015:%.*]] = phi ptr [ [[INCDEC_PTR7:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[WHILE_END]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PSRC]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[PDST]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[PSRC_ADDR_017:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BLKCNT_016:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[PDST_ADDR_015:%.*]] = phi ptr [ [[INCDEC_PTR7:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_017]], i32 4
-; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[PSRC_ADDR_017]], align 4
-; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP9]], 0
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP9]], -2147483648
-; CHECK-NEXT: [[SUB:%.*]] = sub nsw i32 0, [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[PSRC_ADDR_017]], align 4
+; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP12]], 0
+; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP12]], -2147483648
+; CHECK-NEXT: [[SUB:%.*]] = sub nsw i32 0, [[TMP12]]
; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP2]], i32 2147483647, i32 [[SUB]]
-; CHECK-NEXT: [[COND6:%.*]] = select i1 [[CMP1]], i32 [[TMP9]], i32 [[COND]]
+; CHECK-NEXT: [[COND6:%.*]] = select i1 [[CMP1]], i32 [[TMP12]], i32 [[COND]]
; CHECK-NEXT: [[INCDEC_PTR7]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_015]], i32 4
; CHECK-NEXT: store i32 [[COND6]], ptr [[PDST_ADDR_015]], align 4
; CHECK-NEXT: [[DEC]] = add i32 [[BLKCNT_016]], -1
; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: while.end:
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END]], label %[[WHILE_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[WHILE_END]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
index c473123c116b1..2121531d3b0cd 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg -simplifycfg-require-and-preserve-domtree=1 -tail-predication=enabled < %s -S -o - | FileCheck %s
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
@@ -6,21 +6,22 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
; Should not be vectorized
define i64 @add_i64_i64(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i64_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X:%.*]], i32 [[I_08]]
+; CHECK-LABEL: define i64 @add_i64_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[FOR_BODY:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i32 [[I_08]]
; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
; CHECK-NEXT: [[ADD]] = add nsw i64 [[TMP0]], [[R_07]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -45,46 +46,47 @@ for.cond.cleanup:
; 4x to use VADDLV
; FIXME: TailPredicate
define i64 @add_i32_i64(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i32_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @add_i32_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[TMP3]] = add i64 [[VEC_PHI]], [[TMP2]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[X]], i32 [[I_08]]
; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[CONV:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -110,46 +112,47 @@ for.cond.cleanup:
; 4x to use VADDLV
; FIXME: TailPredicate
define i64 @add_i16_i64(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i16_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @add_i16_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[TMP3]] = add i64 [[VEC_PHI]], [[TMP2]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X]], i32 [[I_08]]
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP5]] to i64
; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -175,46 +178,47 @@ for.cond.cleanup:
; 4x to use VADDLV
; FIXME: TailPredicate
define i64 @add_i8_i64(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i8_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @add_i8_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[TMP3]] = add i64 [[VEC_PHI]], [[TMP2]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 [[I_08]]
; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP5]] to i64
; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[R_07]], [[CONV]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -239,27 +243,28 @@ for.cond.cleanup:
; 4x to use VADDV.u32
define i32 @add_i32_i32(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i32_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @add_i32_i32(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> zeroinitializer)
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD]])
; CHECK-NEXT: [[TMP2]] = add i32 [[VEC_PHI]], [[TMP1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[R_0_LCSSA]]
;
entry:
@@ -283,19 +288,20 @@ for.cond.cleanup:
; 8x to use VADDV.u16
define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i16_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @add_i16_i32(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD]] to <8 x i32>
; CHECK-NEXT: [[TMP2:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP1]], <8 x i32> zeroinitializer
@@ -303,9 +309,9 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) #0 {
; CHECK-NEXT: [[TMP4]] = add i32 [[VEC_PHI]], [[TMP3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP4]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[R_0_LCSSA]]
;
entry:
@@ -330,19 +336,20 @@ for.cond.cleanup:
; 16x to use VADDV.u16
define i32 @add_i8_i32(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i8_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @add_i8_i32(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP1]], <16 x i32> zeroinitializer
@@ -350,9 +357,9 @@ define i32 @add_i8_i32(ptr nocapture readonly %x, i32 %n) #0 {
; CHECK-NEXT: [[TMP4]] = add i32 [[VEC_PHI]], [[TMP3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP4]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[R_0_LCSSA]]
;
entry:
@@ -377,27 +384,28 @@ for.cond.cleanup:
; 8x to use VADDV.u16
define signext i16 @add_i16_i16(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i16_i16(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define signext i16 @add_i16_i16(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP8]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> zeroinitializer)
; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[WIDE_MASKED_LOAD]])
; CHECK-NEXT: [[TMP2]] = add i16 [[VEC_PHI]], [[TMP1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i16 [[R_0_LCSSA]]
;
entry:
@@ -421,19 +429,20 @@ for.cond.cleanup:
; 16x to use VADDV.u8
define signext i16 @add_i8_i16(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i8_i16(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define signext i16 @add_i8_i16(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP8]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i16>
; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP1]], <16 x i16> zeroinitializer
@@ -441,9 +450,9 @@ define signext i16 @add_i8_i16(ptr nocapture readonly %x, i32 %n) #0 {
; CHECK-NEXT: [[TMP4]] = add i16 [[VEC_PHI]], [[TMP3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[TMP4]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i16 [[R_0_LCSSA]]
;
entry:
@@ -468,27 +477,28 @@ for.cond.cleanup:
; 16x to use VADDV.u8
define zeroext i8 @add_i8_i8(ptr nocapture readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @add_i8_i8(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP7]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define zeroext i8 @add_i8_i8(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP7]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i8 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i8 [ 0, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> zeroinitializer)
; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[WIDE_MASKED_LOAD]])
; CHECK-NEXT: [[TMP2]] = add i8 [[VEC_PHI]], [[TMP1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i8 [[R_0_LCSSA]]
;
entry:
@@ -512,24 +522,25 @@ for.cond.cleanup:
; Not vectorized
define i64 @mla_i64_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i64_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP8]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X:%.*]], i32 [[I_010]]
+; CHECK-LABEL: define i64 @mla_i64_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i32 [[I_010]]
; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y:%.*]], i32 [[I_010]]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i32 [[I_010]]
; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i64 [[TMP1]], [[TMP0]]
; CHECK-NEXT: [[ADD]] = add nsw i64 [[MUL]], [[R_09]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_010]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -557,22 +568,23 @@ for.cond.cleanup:
; 4x to use VMLAL.u32
; FIXME: TailPredicate
define i64 @mla_i32_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i32_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @mla_i32_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[Y:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[Y]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
@@ -580,17 +592,17 @@ define i64 @mla_i32_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i3
; CHECK-NEXT: [[TMP5]] = add i64 [[VEC_PHI]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[X]], i32 [[I_010]]
; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[Y]], i32 [[I_010]]
@@ -600,9 +612,9 @@ define i64 @mla_i32_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i3
; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_09]], [[CONV]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_010]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -631,55 +643,56 @@ for.cond.cleanup:
; 8x to use VMLAL.u16
; FIXME: TailPredicate
define i64 @mla_i16_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i16_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @mla_i16_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP10]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483640
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD1]] to <8 x i64>
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i16> [[WIDE_LOAD1]] to <8 x i64>
; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <8 x i64> [[TMP4]], [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
-; CHECK-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <8 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
+; CHECK-NEXT: [[TMP6]] = add i64 [[VEC_PHI]], [[TMP5]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_012:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X]], i32 [[I_012]]
-; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
-; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP9]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
+; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP8]] to i32
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y]], i32 [[I_012]]
-; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
-; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP10]] to i32
+; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
+; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP9]] to i32
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
; CHECK-NEXT: [[CONV3:%.*]] = sext i32 [[MUL]] to i64
; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_011]], [[CONV3]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_012]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -710,55 +723,56 @@ for.cond.cleanup:
; 8x to use VMLAL.u16
; FIXME: TailPredicate
define i64 @mla_i8_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i8_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @mla_i8_i64(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP10]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483640
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD1]] to <8 x i64>
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = zext <8 x i8> [[WIDE_LOAD1]] to <8 x i64>
; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-NEXT: [[TMP5:%.*]] = mul nuw nsw <8 x i64> [[TMP4]], [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
-; CHECK-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <8 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
+; CHECK-NEXT: [[TMP6]] = add i64 [[VEC_PHI]], [[TMP5]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_012:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 [[I_012]]
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP9]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP8]] to i32
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i32 [[I_012]]
-; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP10]] to i32
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP9]] to i32
; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
; CHECK-NEXT: [[CONV3:%.*]] = zext nneg i32 [[MUL]] to i64
; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[R_011]], [[CONV3]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_012]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
;
entry:
@@ -788,31 +802,32 @@ for.cond.cleanup:
; 4x to use VMLA.i32
define i32 @mla_i32_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i32_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @mla_i32_i32(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP8]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[Y]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i8], ptr [[Y:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP7]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[TMP5]] = add i32 [[VEC_PHI]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP5]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[R_0_LCSSA]]
;
entry:
@@ -839,33 +854,34 @@ for.cond.cleanup:
; 8x to use VMLA.i16
define i32 @mla_i16_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i16_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @mla_i16_i32(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP2]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
-; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD1]] to <8 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <8 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP1]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD1]] to <8 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <8 x i32> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP4]], <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])
; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP7]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[R_0_LCSSA]]
;
entry:
@@ -894,21 +910,22 @@ for.cond.cleanup:
; 16x to use VMLA.i8
define i32 @mla_i8_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i8_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @mla_i8_i32(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD1]] to <16 x i32>
; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
@@ -918,9 +935,9 @@ define i32 @mla_i8_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32
; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP7]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[R_0_LCSSA]]
;
entry:
@@ -949,31 +966,32 @@ for.cond.cleanup:
; 8x to use VMLA.i16
define signext i16 @mla_i16_i16(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i16_i16(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define signext i16 @mla_i16_i16(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP1]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP7]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i16> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i16> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> [[TMP2]], <8 x i16> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP3]])
; CHECK-NEXT: [[TMP5]] = add i16 [[VEC_PHI]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[TMP5]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i16 [[R_0_LCSSA]]
;
entry:
@@ -1000,33 +1018,34 @@ for.cond.cleanup:
; 16x to use VMLA.i8
define signext i16 @mla_i8_i16(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i8_i16(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define signext i16 @mla_i8_i16(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD1]] to <16 x i16>
-; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i16>
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw <16 x i16> [[TMP3]], [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD1]] to <16 x i16>
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i16>
+; CHECK-NEXT: [[TMP4:%.*]] = mul nuw <16 x i16> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP4]], <16 x i16> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP5]])
; CHECK-NEXT: [[TMP7]] = add i16 [[VEC_PHI]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[TMP7]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i16 [[R_0_LCSSA]]
;
entry:
@@ -1055,31 +1074,32 @@ for.cond.cleanup:
; 16x to use VMLA.i8
define zeroext i8 @mla_i8_i8(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i8_i8(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP10]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define zeroext i8 @mla_i8_i8(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP10]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i8 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i8 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[Y1:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP7]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> [[TMP2]], <16 x i8> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[TMP3]])
; CHECK-NEXT: [[TMP5]] = add i8 [[VEC_PHI]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[TMP5]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i8 [[R_0_LCSSA]]
;
entry:
@@ -1106,33 +1126,34 @@ for.cond.cleanup:
; 8x as different types
define i32 @red_mla_ext_s8_s16_s32(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, i32 %n) #0 {
-; CHECK-LABEL: @red_mla_ext_s8_s16_s32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP9_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @red_mla_ext_s8_s16_s32(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP9_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[B:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP2]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
-; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD1]] to <8 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <8 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP1]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD1]] to <8 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <8 x i32> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP4]], <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])
; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP7]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[S_0_LCSSA]]
;
entry:
@@ -1168,24 +1189,25 @@ for.cond.cleanup:
; 4x as different sext vs zext
define i64 @red_mla_ext_s16_u16_s64(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, i32 %n) #0 {
-; CHECK-LABEL: @red_mla_ext_s16_u16_s64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP9_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @red_mla_ext_s16_u16_s64(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP9_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -4
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[A:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[A]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i8], ptr [[B:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP11]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[WIDE_LOAD1]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]
; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i32> [[TMP4]] to <4 x i64>
@@ -1193,17 +1215,17 @@ define i64 @red_mla_ext_s16_u16_s64(ptr noalias nocapture readonly %A, ptr noali
; CHECK-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[S_010:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_011:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[S_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i8], ptr [[A]], i32 [[I_011]]
; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP9]] to i32
@@ -1215,9 +1237,9 @@ define i64 @red_mla_ext_s16_u16_s64(ptr noalias nocapture readonly %A, ptr noali
; CHECK-NEXT: [[ADD]] = add nsw i64 [[S_010]], [[MUL2]]
; CHECK-NEXT: [[INC]] = add nuw i32 [[I_011]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[S_0_LCSSA]]
;
entry:
@@ -1254,33 +1276,34 @@ for.cond.cleanup:
; 4x as different sext vs zext
define i32 @red_mla_u8_s8_u32(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, i32 %n) #0 {
-; CHECK-LABEL: @red_mla_u8_s8_u32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP9_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @red_mla_u8_s8_u32(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP9_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_MASKED_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_MASKED_LOAD1]] to <4 x i32>
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[B1:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP9]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
-; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[WIDE_MASKED_LOAD2]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[WIDE_MASKED_LOAD1]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP7]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[S_0_LCSSA]]
;
entry:
@@ -1316,47 +1339,48 @@ for.cond.cleanup:
; Make sure interleave group members feeding in-loop reductions can be handled.
define i32 @reduction_interleave_group(i32 %n, ptr %arr) #0 {
-; CHECK-LABEL: @reduction_interleave_group(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GUARD:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[GUARD]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i32 @reduction_interleave_group(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[ARR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[GUARD:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[GUARD]], label %[[FOR_BODY_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -1
; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[TMP1]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 7
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP2]], -4
-; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC]], 1
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[N_VEC]], 1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[DOTIDX:%.*]] = shl i32 [[INDEX]], 3
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[ARR:%.*]], i32 [[DOTIDX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[DOTIDX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC1]])
-; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[VEC_PHI]], [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC]])
-; CHECK-NEXT: [[TMP9]] = add i32 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC1]])
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[VEC_PHI]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC]])
+; CHECK-NEXT: [[TMP8]] = add i32 [[TMP6]], [[TMP7]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[RED_PHI:%.*]] = phi i32 [ [[RED_2:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [4 x i8], ptr [[ARR]], i32 [[IV]]
-; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr i8, ptr [[TMP11]], i32 4
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[RED_PHI:%.*]] = phi i32 [ [[RED_2:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [4 x i8], ptr [[ARR]], i32 [[IV]]
+; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr i8, ptr [[TMP10]], i32 4
; CHECK-NEXT: [[L_0:%.*]] = load i32, ptr [[GEP_0]], align 4
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds [4 x i8], ptr [[ARR]], i32 [[IV]]
; CHECK-NEXT: [[L_1:%.*]] = load i32, ptr [[GEP_1]], align 4
@@ -1364,9 +1388,9 @@ define i32 @reduction_interleave_group(i32 %n, ptr %arr) #0 {
; CHECK-NEXT: [[RED_2]] = add i32 [[RED_1]], [[L_1]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 2
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK: exit:
-; CHECK-NEXT: [[RET_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RED_2]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RET_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RED_2]], %[[FOR_BODY]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RET_LCSSA]]
;
entry:
@@ -1394,30 +1418,31 @@ exit:
; 16x to use VMLA.i8, same as mla_i8_i32 with multiple uses of the ext `add(mul(x, x))`
define i32 @mla_i8_i32_multiuse(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
-; CHECK-LABEL: @mla_i8_i32_multiuse(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define i32 @mla_i8_i32_multiuse(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
-; CHECK-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
-; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw <16 x i32> [[TMP7]], [[TMP7]]
+; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw <16 x i32> [[TMP1]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP2]], <16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])
; CHECK-NEXT: [[TMP5]] = add i32 [[VEC_PHI]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_COND_CLEANUP]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP5]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[R_0_LCSSA]]
;
entry:
@@ -1442,50 +1467,51 @@ for.cond.cleanup:
}
define i64 @mla_xx_sext_zext(ptr nocapture noundef readonly %x, i32 %n) #0 {
-; CHECK-LABEL: @mla_xx_sext_zext(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP9]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @mla_xx_sext_zext(
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483640
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-NEXT: [[TMP3:%.*]] = mul nsw <8 x i64> [[TMP1]], [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP3]])
-; CHECK-NEXT: [[TMP5]] = add i64 [[VEC_PHI]], [[TMP4]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP1]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP2]])
+; CHECK-NEXT: [[TMP4]] = add i64 [[VEC_PHI]], [[TMP3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[S_0_LCSSA]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[S_010:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_011:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[S_010:%.*]] = phi i64 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X]], i32 [[I_011]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
-; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP7]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
+; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP6]] to i32
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV]], [[CONV]]
; CHECK-NEXT: [[CONV3:%.*]] = zext nneg i32 [[MUL]] to i64
; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[S_010]], [[CONV3]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_011]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
;
entry:
%cmp9 = icmp sgt i32 %n, 0
@@ -1510,49 +1536,50 @@ for.body:
}
define i64 @mla_and_add_together_16_64(ptr nocapture noundef readonly %x, i32 noundef %n) #0 {
-; CHECK-LABEL: @mla_and_add_together_16_64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-LABEL: define i64 @mla_and_add_together_16_64(
+; CHECK-SAME: ptr noundef readonly captures(none) [[X:%.*]], i32 noundef [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP16]])
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i32 [[N]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483640
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP4]], [[TMP4]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP1]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP2]])
-; CHECK-NEXT: [[TMP5]] = add i64 [[VEC_PHI]], [[TMP3]]
-; CHECK-NEXT: [[TMP10:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP10]])
+; CHECK-NEXT: [[TMP4]] = add i64 [[VEC_PHI]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])
; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI1]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[ADD6_LCSSA:%.*]] = phi i32 [ [[ADD6:%.*]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[ADD6_LCSSA:%.*]] = phi i32 [ [[ADD6:%.*]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[CONV7:%.*]] = sext i32 [[ADD6_LCSSA]] to i64
; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 [[CONV7]], [[ADD_LCSSA]]
; CHECK-NEXT: ret i64 [[DIV]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_019:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[T_018:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[S_017:%.*]] = phi i32 [ [[ADD6]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_019:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[T_018:%.*]] = phi i64 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[S_017:%.*]] = phi i32 [ [[ADD6]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X]], i32 [[I_019]]
; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP9]] to i32
@@ -1562,7 +1589,7 @@ define i64 @mla_and_add_together_16_64(ptr nocapture noundef readonly %x, i32 no
; CHECK-NEXT: [[ADD6]] = add nsw i32 [[S_017]], [[CONV]]
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_019]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
;
entry:
%cmp16 = icmp sgt i32 %n, 0
@@ -1591,20 +1618,21 @@ for.body:
}
define i64 @interleave_doublereduct_i16_i64(ptr %x, ptr %y, i32 %n) {
-; CHECK-LABEL: @interleave_doublereduct_i16_i64(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP23:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP23]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[T_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD12:%.*]], [[FOR_BODY]] ]
+; CHECK-LABEL: define i64 @interleave_doublereduct_i16_i64(
+; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP23:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP23]], label %[[FOR_BODY:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[T_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD12:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: ret i64 [[T_0_LCSSA]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_025:%.*]] = phi i32 [ [[ADD13:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[T_024:%.*]] = phi i64 [ [[ADD12]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i32 [[I_025]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_025:%.*]] = phi i32 [ [[ADD13:%.*]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[T_024:%.*]] = phi i64 [ [[ADD12]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X]], i32 [[I_025]]
; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP0]] to i32
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i32 [[I_025]]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y]], i32 [[I_025]]
; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP1]] to i32
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
@@ -1622,7 +1650,7 @@ define i64 @interleave_doublereduct_i16_i64(ptr %x, ptr %y, i32 %n) {
; CHECK-NEXT: [[ADD12]] = add nsw i64 [[ADD]], [[CONV11]]
; CHECK-NEXT: [[ADD13]] = add nuw nsw i32 [[I_025]], 2
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[ADD13]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP]]
;
entry:
%cmp23 = icmp sgt i32 %n, 0
@@ -1660,58 +1688,59 @@ for.body:
}
define i64 @test_std_q31(ptr %x, i32 %n) #0 {
-; CHECK-LABEL: @test_std_q31(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-LABEL: define i64 @test_std_q31(
+; CHECK-SAME: ptr [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP11]])
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i32 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644
-; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY1]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[FOR_BODY1]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[FOR_BODY1]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[X:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[X]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[WIDE_LOAD]], splat (i32 8)
; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[TMP1]] to <4 x i64>
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP2]])
; CHECK-NEXT: [[TMP4]] = add i64 [[VEC_PHI]], [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = mul nsw <4 x i64> [[TMP2]], [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP6]])
-; CHECK-NEXT: [[TMP8]] = add i64 [[VEC_PHI1]], [[TMP7]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <4 x i64> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
+; CHECK-NEXT: [[TMP7]] = add i64 [[VEC_PHI1]], [[TMP6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY1]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[ADD:%.*]] = phi i64 [ [[ADD1:%.*]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[ADD3:%.*]] = phi i64 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 [[ADD3]], [[ADD]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[ADD3_LCSSA:%.*]] = phi i64 [ [[ADD3:%.*]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 [[ADD3_LCSSA]], [[ADD_LCSSA]]
; CHECK-NEXT: ret i64 [[DIV]]
-; CHECK: for.body:
-; CHECK-NEXT: [[S_014:%.*]] = phi i64 [ [[ADD1]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[I_013:%.*]] = phi i32 [ [[ADD4:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[T_012:%.*]] = phi i64 [ [[ADD5]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[S_014:%.*]] = phi i64 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[I_013:%.*]] = phi i32 [ [[ADD4:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[T_012:%.*]] = phi i64 [ [[ADD3]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[X]], i32 [[I_013]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[SHR:%.*]] = ashr i32 [[TMP0]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[SHR:%.*]] = ashr i32 [[TMP9]], 8
; CHECK-NEXT: [[CONV:%.*]] = sext i32 [[SHR]] to i64
-; CHECK-NEXT: [[ADD1]] = add nsw i64 [[S_014]], [[CONV]]
+; CHECK-NEXT: [[ADD]] = add nsw i64 [[S_014]], [[CONV]]
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV]]
-; CHECK-NEXT: [[ADD5]] = add nuw nsw i64 [[MUL]], [[T_012]]
+; CHECK-NEXT: [[ADD3]] = add nuw nsw i64 [[MUL]], [[T_012]]
; CHECK-NEXT: [[ADD4]] = add nuw nsw i32 [[I_013]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[ADD4]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
;
entry:
%cmp11 = icmp sgt i32 %n, 0
@@ -1739,30 +1768,31 @@ for.body:
}
define i64 @test_fir_q15(ptr %x, ptr %y, i32 %n) #0 {
-; CHECK-LABEL: @test_fir_q15(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP23:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP23]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.body.preheader:
+; CHECK-LABEL: define i64 @test_fir_q15(
+; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP23:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP23]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -1
; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[TMP1]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 15
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP2]], -8
-; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC]], 1
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[N_VEC]], 1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP3]], align 2
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i16>, ptr [[TMP4]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
; CHECK-NEXT: [[TMP6:%.*]] = sext <8 x i16> [[STRIDED_VEC3]] to <8 x i64>
@@ -1773,46 +1803,46 @@ define i64 @test_fir_q15(ptr %x, ptr %y, i32 %n) #0 {
; CHECK-NEXT: [[TMP11:%.*]] = sext <8 x i16> [[STRIDED_VEC4]] to <8 x i64>
; CHECK-NEXT: [[TMP12:%.*]] = sext <8 x i16> [[STRIDED_VEC1]] to <8 x i64>
; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <8 x i64> [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP13]])
-; CHECK-NEXT: [[TMP16]] = add i64 [[TMP10]], [[TMP15]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP13]])
+; CHECK-NEXT: [[TMP15]] = add i64 [[TMP10]], [[TMP14]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD12:%.*]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD12:%.*]], %[[FOR_BODY]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[S_0_LCSSA]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I_025:%.*]] = phi i32 [ [[ADD13:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[S_024:%.*]] = phi i64 [ [[ADD12]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I_025:%.*]] = phi i32 [ [[ADD13:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[S_024:%.*]] = phi i64 [ [[ADD12]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[I_025]]
-; CHECK-NEXT: [[TMP18:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
-; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP18]] to i32
+; CHECK-NEXT: [[TMP17:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
+; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP17]] to i32
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i32 [[I_025]]
-; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
-; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP19]] to i32
+; CHECK-NEXT: [[TMP18:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
+; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP18]] to i32
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
; CHECK-NEXT: [[CONV3:%.*]] = sext i32 [[MUL]] to i64
; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[S_024]], [[CONV3]]
; CHECK-NEXT: [[ADD4:%.*]] = or disjoint i32 [[I_025]], 1
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i32 [[ADD4]]
-; CHECK-NEXT: [[TMP20:%.*]] = load i16, ptr [[ARRAYIDX5]], align 2
-; CHECK-NEXT: [[CONV6:%.*]] = sext i16 [[TMP20]] to i32
+; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX5]], align 2
+; CHECK-NEXT: [[CONV6:%.*]] = sext i16 [[TMP19]] to i32
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i32 [[ADD4]]
-; CHECK-NEXT: [[TMP21:%.*]] = load i16, ptr [[ARRAYIDX8]], align 2
-; CHECK-NEXT: [[CONV9:%.*]] = sext i16 [[TMP21]] to i32
+; CHECK-NEXT: [[TMP20:%.*]] = load i16, ptr [[ARRAYIDX8]], align 2
+; CHECK-NEXT: [[CONV9:%.*]] = sext i16 [[TMP20]] to i32
; CHECK-NEXT: [[MUL10:%.*]] = mul nsw i32 [[CONV9]], [[CONV6]]
; CHECK-NEXT: [[CONV11:%.*]] = sext i32 [[MUL10]] to i64
; CHECK-NEXT: [[ADD12]] = add nsw i64 [[ADD]], [[CONV11]]
; CHECK-NEXT: [[ADD13]] = add nuw nsw i32 [[I_025]], 2
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[ADD13]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], !llvm.loop [[LOOP40:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP40:![0-9]+]]
;
entry:
%cmp23 = icmp sgt i32 %n, 0
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
index 5fcc7e08ac9c1..6424846a99c7f 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg < %s -S -o - | FileCheck %s --check-prefix=CHECK
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-COST
; REQUIRES: asserts
@@ -9,30 +9,31 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
; CHECK-COST-LABEL: arm_offset_q15
define void @arm_offset_q15(ptr nocapture readonly %pSrc, i16 signext %offset, ptr nocapture noalias %pDst, i32 %blockSize) #0 {
-; CHECK-LABEL: @arm_offset_q15(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP_NOT6:%.*]] = icmp eq i32 [[BLOCKSIZE:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT6]], label [[WHILE_END:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define void @arm_offset_q15(
+; CHECK-SAME: ptr readonly captures(none) [[PSRC:%.*]], i16 signext [[OFFSET:%.*]], ptr noalias captures(none) [[PDST:%.*]], i32 [[BLOCKSIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP_NOT6:%.*]] = icmp eq i32 [[BLOCKSIZE]], 0
+; CHECK-NEXT: br i1 [[CMP_NOT6]], label %[[WHILE_END:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[BLOCKSIZE]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <8 x i16> poison, i16 [[OFFSET:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT7]], <8 x i16> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[OFFSET]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC:%.*]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX5:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[PDST:%.*]], i32 [[OFFSET_IDX5]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[BLOCKSIZE]])
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[NEXT_GEP]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
-; CHECK-NEXT: [[TMP0:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[WIDE_MASKED_LOAD]], <8 x i16> [[BROADCAST_SPLAT8]])
-; CHECK-NEXT: call void @llvm.masked.store.v8i16.p0(<8 x i16> [[TMP0]], ptr align 2 [[NEXT_GEP6]], <8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP0:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[WIDE_MASKED_LOAD]], <8 x i16> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: call void @llvm.masked.store.v8i16.p0(<8 x i16> [[TMP0]], ptr align 2 [[NEXT_GEP2]], <8 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP1]], label [[WHILE_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: while.end:
+; CHECK-NEXT: br i1 [[TMP1]], label %[[WHILE_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[WHILE_END]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll
index b533de44b526a..62066d65ea6f0 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg < %s -S -o - | FileCheck %s --check-prefix=CHECK
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-COST
; REQUIRES: asserts
@@ -12,83 +12,84 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
; CHECK-COST: Cost of 2 for VF 4: WIDEN ir<%or.cond> = select ir<%cmp2>, ir<true>, ir<%cmp3>
define float @test(ptr nocapture readonly %pA, ptr nocapture readonly %pB, i32 %blockSize) #0 {
-; CHECK-LABEL: @test(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CMP_NOT16:%.*]] = icmp eq i32 [[BLOCKSIZE:%.*]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT16]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
-; CHECK: while.body.preheader:
+; CHECK-LABEL: define float @test(
+; CHECK-SAME: ptr readonly captures(none) [[PA:%.*]], ptr readonly captures(none) [[PB:%.*]], i32 [[BLOCKSIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP_NOT16:%.*]] = icmp eq i32 [[BLOCKSIZE]], 0
+; CHECK-NEXT: br i1 [[CMP_NOT16]], label %[[WHILE_END:.*]], label %[[WHILE_BODY_PREHEADER:.*]]
+; CHECK: [[WHILE_BODY_PREHEADER]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[BLOCKSIZE]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[BLOCKSIZE]], -4
; CHECK-NEXT: [[TMP0:%.*]] = shl i32 [[N_VEC]], 2
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PA:%.*]], i32 [[TMP0]]
-; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[N_VEC]], 2
-; CHECK-NEXT: [[IND_END1:%.*]] = getelementptr i8, ptr [[PB:%.*]], i32 [[TMP1]]
-; CHECK-NEXT: [[IND_END3:%.*]] = and i32 [[BLOCKSIZE]], 3
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PA]], i32 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PB]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[BLOCKSIZE]], 3
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PA]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX5:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[PB]], i32 [[OFFSET_IDX5]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PB]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[NEXT_GEP6]], align 4
-; CHECK-NEXT: [[TMP18:%.*]] = fcmp fast une <4 x float> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = fcmp fast une <4 x float> [[WIDE_LOAD7]], zeroinitializer
-; CHECK-NEXT: [[TMP20:%.*]] = select <4 x i1> [[TMP18]], <4 x i1> splat (i1 true), <4 x i1> [[TMP19]]
-; CHECK-NEXT: [[TMP4:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP5:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD7]])
-; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <4 x float> [[TMP5]], [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = fsub fast <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD7]]
-; CHECK-NEXT: [[TMP8:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = fdiv fast <4 x float> [[TMP8]], [[TMP6]]
-; CHECK-NEXT: [[TMP10:%.*]] = select ninf <4 x i1> [[TMP20]], <4 x float> [[TMP9]], <4 x float> splat (float -0.000000e+00)
-; CHECK-NEXT: [[PREDPHI]] = fadd reassoc arcp contract afn <4 x float> [[VEC_PHI]], [[TMP10]]
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[NEXT_GEP2]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp fast une <4 x float> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp fast une <4 x float> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP9:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD3]])
+; CHECK-NEXT: [[TMP10:%.*]] = fadd fast <4 x float> [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = fsub fast <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP12:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP11]])
+; CHECK-NEXT: [[TMP13:%.*]] = fdiv fast <4 x float> [[TMP12]], [[TMP10]]
+; CHECK-NEXT: [[TMP14:%.*]] = select ninf <4 x i1> [[TMP7]], <4 x float> [[TMP13]], <4 x float> splat (float -0.000000e+00)
+; CHECK-NEXT: [[PREDPHI]] = fadd reassoc arcp contract afn <4 x float> [[VEC_PHI]], [[TMP14]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI]])
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[BLOCKSIZE]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[WHILE_END]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[PA]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[PB]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
-; CHECK: while.body:
-; CHECK-NEXT: [[PA_ADDR_020:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[IF_END:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[PB_ADDR_019:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[BLOCKSIZE_ADDR_018:%.*]] = phi i32 [ [[DEC:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[ACCUM_017:%.*]] = phi float [ [[ACCUM_1:%.*]], [[IF_END]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[WHILE_END]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[PA]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[PB]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[BLOCKSIZE]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[PA_ADDR_020:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[IF_END:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[PB_ADDR_019:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[IF_END]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BLOCKSIZE_ADDR_018:%.*]] = phi i32 [ [[DEC:%.*]], %[[IF_END]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ACCUM_017:%.*]] = phi float [ [[ACCUM_1:%.*]], %[[IF_END]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PA_ADDR_020]], i32 4
-; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[PA_ADDR_020]], align 4
+; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[PA_ADDR_020]], align 4
; CHECK-NEXT: [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[PB_ADDR_019]], i32 4
-; CHECK-NEXT: [[TMP14:%.*]] = load float, ptr [[PB_ADDR_019]], align 4
-; CHECK-NEXT: [[CMP2:%.*]] = fcmp fast une float [[TMP13]], 0.000000e+00
-; CHECK-NEXT: [[CMP3:%.*]] = fcmp fast une float [[TMP14]], 0.000000e+00
+; CHECK-NEXT: [[TMP18:%.*]] = load float, ptr [[PB_ADDR_019]], align 4
+; CHECK-NEXT: [[CMP2:%.*]] = fcmp fast une float [[TMP17]], 0.000000e+00
+; CHECK-NEXT: [[CMP3:%.*]] = fcmp fast une float [[TMP18]], 0.000000e+00
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[CMP2]], i1 true, i1 [[CMP3]]
-; CHECK-NEXT: br i1 [[OR_COND]], label [[IF_THEN:%.*]], label [[IF_END]]
-; CHECK: if.then:
-; CHECK-NEXT: [[TMP15:%.*]] = tail call fast float @llvm.fabs.f32(float [[TMP13]])
-; CHECK-NEXT: [[TMP16:%.*]] = tail call fast float @llvm.fabs.f32(float [[TMP14]])
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP16]], [[TMP15]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub fast float [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = tail call fast float @llvm.fabs.f32(float [[SUB]])
-; CHECK-NEXT: [[DIV:%.*]] = fdiv fast float [[TMP17]], [[ADD]]
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[IF_THEN:.*]], label %[[IF_END]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: [[TMP19:%.*]] = tail call fast float @llvm.fabs.f32(float [[TMP17]])
+; CHECK-NEXT: [[TMP20:%.*]] = tail call fast float @llvm.fabs.f32(float [[TMP18]])
+; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP20]], [[TMP19]]
+; CHECK-NEXT: [[SUB:%.*]] = fsub fast float [[TMP17]], [[TMP18]]
+; CHECK-NEXT: [[TMP21:%.*]] = tail call fast float @llvm.fabs.f32(float [[SUB]])
+; CHECK-NEXT: [[DIV:%.*]] = fdiv fast float [[TMP21]], [[ADD]]
; CHECK-NEXT: [[ADD4:%.*]] = fadd fast float [[DIV]], [[ACCUM_017]]
-; CHECK-NEXT: br label [[IF_END]]
-; CHECK: if.end:
-; CHECK-NEXT: [[ACCUM_1]] = phi float [ [[ADD4]], [[IF_THEN]] ], [ [[ACCUM_017]], [[WHILE_BODY]] ]
+; CHECK-NEXT: br label %[[IF_END]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: [[ACCUM_1]] = phi float [ [[ADD4]], %[[IF_THEN]] ], [ [[ACCUM_017]], %[[WHILE_BODY]] ]
; CHECK-NEXT: [[DEC]] = add i32 [[BLOCKSIZE_ADDR_018]], -1
; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: while.end:
-; CHECK-NEXT: [[ACCUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[ACCUM_1]], [[IF_END]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END]], label %[[WHILE_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: [[ACCUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ACCUM_1]], %[[IF_END]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[ACCUM_0_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
index 54d698e56a494..0a349bed2beb5 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
@@ -1,28 +1,29 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg -S -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp,mve4beat -simplifycfg-require-and-preserve-domtree=1 -tail-predication=disabled < %s | FileCheck %s
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-none-none-eabi"
define hidden void @pointer_phi_v4i32_add1(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %s, i32%y) {
-; CHECK-LABEL: @pointer_phi_v4i32_add1(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4i32_add1(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[S:%.*]], i32 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP1]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[TMP1]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -44,39 +45,40 @@ end:
}
define hidden void @pointer_phi_v4i32_add2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v4i32_add2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 7968
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 3984
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4i32_add2(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 7968
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 3984
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 3
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 996
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[A_ADDR_09]], align 4
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 996, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[A_ADDR_09]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 8
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[Y]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP4]], [[Y]]
; CHECK-NEXT: store i32 [[ADD]], ptr [[B_ADDR_07]], align 4
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -98,39 +100,40 @@ end:
}
define hidden void @pointer_phi_v4i32_add3(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v4i32_add3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 11952
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 3984
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4i32_add3(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 11952
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 3984
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 12, i32 24, i32 36>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 12, i32 24, i32 36>
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[VECTOR_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 48
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 996
-; CHECK-NEXT: br i1 [[TMP4]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[A_ADDR_09]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 996
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 996, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[A_ADDR_09]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 12
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP3]], [[Y]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP4]], [[Y]]
; CHECK-NEXT: store i32 [[ADD]], ptr [[B_ADDR_07]], align 4
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -152,25 +155,26 @@ end:
}
define hidden void @pointer_phi_v8i16_add1(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v8i16_add1(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y:%.*]] to i16
+; CHECK-LABEL: define hidden void @pointer_phi_v8i16_add1(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y]] to i16
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <8 x i16> [[TMP1]], ptr [[NEXT_GEP5]], align 2
+; CHECK-NEXT: store <8 x i16> [[TMP1]], ptr [[NEXT_GEP2]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP2]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[TMP2]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -193,31 +197,32 @@ end:
}
define hidden void @pointer_phi_v8i16_add2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v8i16_add2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y:%.*]] to i16
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 3968
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 1984
+; CHECK-LABEL: define hidden void @pointer_phi_v8i16_add2(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y]] to i16
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i32 3968
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B]], i32 1984
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[NEXT_GEP]], align 2
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i16> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <8 x i16> [[TMP1]], ptr [[NEXT_GEP5]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i16> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <8 x i16> [[TMP3]], ptr [[NEXT_GEP2]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP4]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_011:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_09:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_011:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 992, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_09:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[L1:%.*]] = load i16, ptr [[A_ADDR_011]], align 2
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_011]], i32 4
; CHECK-NEXT: [[CONV1:%.*]] = add i16 [[L1]], [[TMP0]]
@@ -225,8 +230,8 @@ define hidden void @pointer_phi_v8i16_add2(ptr noalias nocapture readonly %A, pt
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_09]], i32 2
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_010]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -249,14 +254,15 @@ end:
}
define hidden void @pointer_phi_v8i16_add3(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v8i16_add3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y:%.*]] to i16
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_011:%.*]] = phi ptr [ [[A:%.*]], [[ENTRY:%.*]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_09:%.*]] = phi ptr [ [[B:%.*]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-LABEL: define hidden void @pointer_phi_v8i16_add3(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y]] to i16
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_011:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_09:%.*]] = phi ptr [ [[B]], %[[ENTRY]] ], [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[L1:%.*]] = load i16, ptr [[A_ADDR_011]], align 2
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_011]], i32 6
; CHECK-NEXT: [[CONV1:%.*]] = add i16 [[L1]], [[TMP0]]
@@ -264,8 +270,8 @@ define hidden void @pointer_phi_v8i16_add3(ptr noalias nocapture readonly %A, pt
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_09]], i32 2
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_010]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -288,37 +294,38 @@ end:
}
define hidden void @pointer_phi_v16i8_add1(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v16i8_add1(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y:%.*]] to i8
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 992
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 992
+; CHECK-LABEL: define hidden void @pointer_phi_v16i8_add1(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y]] to i8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i32 992
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B]], i32 992
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
-; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = add <16 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <16 x i8> [[TMP1]], ptr [[NEXT_GEP4]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <16 x i8> [[TMP3]], ptr [[NEXT_GEP1]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_010:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_09:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
+; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_010:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_09:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 992, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_010]], i32 1
-; CHECK-NEXT: [[CONV1:%.*]] = add i8 [[TMP3]], [[TMP0]]
+; CHECK-NEXT: [[CONV1:%.*]] = add i8 [[TMP5]], [[TMP0]]
; CHECK-NEXT: store i8 [[CONV1]], ptr [[B_ADDR_08]], align 1
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_08]], i32 1
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_09]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -342,39 +349,40 @@ end:
}
define hidden void @pointer_phi_v16i8_add2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v16i8_add2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y:%.*]] to i8
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 1984
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 992
+; CHECK-LABEL: define hidden void @pointer_phi_v16i8_add2(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y]] to i8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i32 1984
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B]], i32 992
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[NEXT_GEP]], align 1
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT: [[TMP1:%.*]] = add <16 x i8> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <16 x i8> [[TMP1]], ptr [[NEXT_GEP4]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i8> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <16 x i8> [[TMP3]], ptr [[NEXT_GEP1]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_010:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_09:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
+; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_010:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_09:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 992, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP2]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_010]], i32 2
-; CHECK-NEXT: [[CONV1:%.*]] = add i8 [[TMP3]], [[TMP0]]
+; CHECK-NEXT: [[CONV1:%.*]] = add i8 [[TMP5]], [[TMP0]]
; CHECK-NEXT: store i8 [[CONV1]], ptr [[B_ADDR_08]], align 1
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_08]], i32 1
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_09]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -398,14 +406,15 @@ end:
}
define hidden void @pointer_phi_v16i8_add3(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v16i8_add3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y:%.*]] to i8
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_010:%.*]] = phi ptr [ [[A:%.*]], [[ENTRY:%.*]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[I_09:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_08:%.*]] = phi ptr [ [[B:%.*]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-LABEL: define hidden void @pointer_phi_v16i8_add3(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[Y]] to i8
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_010:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[I_09:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_08:%.*]] = phi ptr [ [[B]], %[[ENTRY]] ], [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_010]], i32 3
; CHECK-NEXT: [[CONV1:%.*]] = add i8 [[TMP1]], [[TMP0]]
@@ -413,8 +422,8 @@ define hidden void @pointer_phi_v16i8_add3(ptr noalias nocapture readonly %A, pt
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_08]], i32 1
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_09]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -438,24 +447,25 @@ end:
}
define hidden void @pointer_phi_v4f32_add1(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, float %y) {
-; CHECK-LABEL: @pointer_phi_v4f32_add1(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4f32_add1(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], float [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <4 x float> [[TMP0]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP0]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP1]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[TMP1]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -477,39 +487,40 @@ end:
}
define hidden void @pointer_phi_v4f32_add2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, float %y) {
-; CHECK-LABEL: @pointer_phi_v4f32_add2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 7968
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 3984
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4f32_add2(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], float [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 7968
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 3984
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 3
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[TMP0:%.*]] = fadd fast <4 x float> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <4 x float> [[TMP0]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 996
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[A_ADDR_09]], align 4
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 996, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[A_ADDR_09]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 8
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP2]], [[Y]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP4]], [[Y]]
; CHECK-NEXT: store float [[ADD]], ptr [[B_ADDR_07]], align 4
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -531,39 +542,40 @@ end:
}
define hidden void @pointer_phi_v4f32_add3(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, float %y) {
-; CHECK-LABEL: @pointer_phi_v4f32_add3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 11952
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 3984
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4f32_add3(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], float [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 11952
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 3984
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 12, i32 24, i32 36>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 12, i32 24, i32 36>
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x float> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = fadd fast <4 x float> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[VECTOR_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 48
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 996
-; CHECK-NEXT: br i1 [[TMP4]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[A_ADDR_09]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 996
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 996, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[A_ADDR_09]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 12
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP3]], [[Y]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP4]], [[Y]]
; CHECK-NEXT: store float [[ADD]], ptr [[B_ADDR_07]], align 4
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -585,24 +597,25 @@ end:
}
define hidden void @pointer_phi_v4half_add1(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, half %y) {
-; CHECK-LABEL: @pointer_phi_v4half_add1(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4half_add1(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], half [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x half> [[BROADCAST_SPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = fadd fast <8 x half> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <8 x half> [[TMP0]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT: store <8 x half> [[TMP0]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP1]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[TMP1]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -624,39 +637,40 @@ end:
}
define hidden void @pointer_phi_v4half_add2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, half %y) {
-; CHECK-LABEL: @pointer_phi_v4half_add2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 3968
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 1984
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4half_add2(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], half [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 3968
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 1984
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x half> [[BROADCAST_SPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x half>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x half> [[WIDE_VEC]], <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[TMP0:%.*]] = fadd fast <8 x half> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <8 x half> [[TMP0]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <8 x half> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <8 x half> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load half, ptr [[A_ADDR_09]], align 4
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 992, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = load half, ptr [[A_ADDR_09]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 4
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast half [[TMP2]], [[Y]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd fast half [[TMP4]], [[Y]]
; CHECK-NEXT: store half [[ADD]], ptr [[B_ADDR_07]], align 4
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 2
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -678,39 +692,40 @@ end:
}
define hidden void @pointer_phi_v4half_add3(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, half %y) {
-; CHECK-LABEL: @pointer_phi_v4half_add3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 5952
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 1984
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4half_add3(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], half [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 5952
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 1984
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x half> [[BROADCAST_SPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i32 [[INDEX]], 6
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX4]]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <24 x half>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <24 x half> [[WIDE_VEC]], <24 x half> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CHECK-NEXT: [[TMP0:%.*]] = fadd fast <8 x half> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: store <8 x half> [[TMP0]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <8 x half> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: store <8 x half> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load half, ptr [[A_ADDR_09]], align 4
+; CHECK-NEXT: br i1 [[TMP3]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_09:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 992, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = load half, ptr [[A_ADDR_09]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 6
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast half [[TMP2]], [[Y]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd fast half [[TMP4]], [[Y]]
; CHECK-NEXT: store half [[ADD]], ptr [[B_ADDR_07]], align 4
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 2
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -735,37 +750,38 @@ end:
!1 = !{!"llvm.loop.interleave.count", i32 2}
define hidden void @pointer_phi_v4i32_uf2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %n, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v4i32_uf2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 239808
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 39968
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4i32_uf2(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[N:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 239808
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 39968
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 24, i32 48, i32 72>
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 96
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 24, i32 48, i32 72>
+; CHECK-NEXT: [[STEP_ADD2:%.*]] = getelementptr i8, <4 x ptr> [[VECTOR_GEP]], i32 96
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP1]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[VECTOR_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[STEP_ADD2]], <4 x i1> splat (i1 true), <4 x i32> poison)
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER1]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 16
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 192
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 9992
-; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
-; CHECK: for.cond.cleanup:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
; CHECK-NEXT: ret void
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_08:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 9992, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_08:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_07:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 9992, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[A_ADDR_08]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_08]], i32 24
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP6]], [[Y]]
@@ -773,7 +789,7 @@ define hidden void @pointer_phi_v4i32_uf2(ptr noalias nocapture readonly %A, ptr
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_06]], i32 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_07]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 10000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
;
entry:
@@ -800,55 +816,56 @@ for.body:
!3 = !{!"llvm.loop.interleave.count", i32 4}
define hidden void @pointer_phi_v4i32_uf4(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i32 %n, i32 %y) {
-; CHECK-LABEL: @pointer_phi_v4i32_uf4(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 239616
-; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 39936
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 0
+; CHECK-LABEL: define hidden void @pointer_phi_v4i32_uf4(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[N:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i32 239616
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i32 39936
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 24, i32 48, i32 72>
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 96
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 192
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 288
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 24, i32 48, i32 72>
+; CHECK-NEXT: [[STEP_ADD4:%.*]] = getelementptr i8, <4 x ptr> [[VECTOR_GEP]], i32 96
+; CHECK-NEXT: [[STEP_ADD_25:%.*]] = getelementptr i8, <4 x ptr> [[VECTOR_GEP]], i32 192
+; CHECK-NEXT: [[STEP_ADD_36:%.*]] = getelementptr i8, <4 x ptr> [[VECTOR_GEP]], i32 288
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP1]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER8:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP2]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP3]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER7]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER8]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER9]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 16
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 32
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 48
-; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[VECTOR_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[STEP_ADD4]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[STEP_ADD_25]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[STEP_ADD_36]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_GATHER3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 16
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 32
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 48
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP6]], align 4
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP7]], align 4
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP8]], align 4
-; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP9]], align 4
-; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP10]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 384
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], 9984
-; CHECK-NEXT: br i1 [[TMP11]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
-; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 9984
+; CHECK-NEXT: br i1 [[TMP9]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
; CHECK-NEXT: ret void
-; CHECK: for.body:
-; CHECK-NEXT: [[A_ADDR_08:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 9984, [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[B_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[A_ADDR_08]], align 4
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[A_ADDR_08:%.*]] = phi ptr [ [[ADD_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP0]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I_07:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 9984, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[B_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[FOR_BODY]] ], [ [[TMP1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[A_ADDR_08]], align 4
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_08]], i32 24
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP12]], [[Y]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP10]], [[Y]]
; CHECK-NEXT: store i32 [[ADD]], ptr [[B_ADDR_06]], align 4
; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_06]], i32 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_07]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 10000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
;
entry:
br label %for.body
@@ -871,51 +888,52 @@ for.body:
}
define hidden void @mult_ptr_iv(ptr noalias nocapture readonly %x, ptr noalias nocapture %z) {
-; CHECK-LABEL: @mult_ptr_iv(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Z:%.*]], i32 3000
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[X:%.*]], i32 3000
+; CHECK-LABEL: define hidden void @mult_ptr_iv(
+; CHECK-SAME: ptr noalias readonly captures(none) [[X:%.*]], ptr noalias captures(none) [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Z]], i32 3000
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[X]], i32 3000
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[Z]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[X]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ], [ [[X]], [[ENTRY]] ]
-; CHECK-NEXT: [[POINTER_PHI5:%.*]] = phi ptr [ [[PTR_IND6:%.*]], [[VECTOR_BODY]] ], [ [[Z]], [[ENTRY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[POINTER_PHI5]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP0]], i32 1
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP0]], <4 x i1> splat (i1 true), <4 x i8> poison), !alias.scope [[META28:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP0]], i32 2
-; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP2]], <4 x i1> splat (i1 true), <4 x i8> poison), !alias.scope [[META28]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER8:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP3]], <4 x i1> splat (i1 true), <4 x i8> poison), !alias.scope [[META28]]
-; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], splat (i8 10)
-; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER8]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP1]], i32 1
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP4]], <4 x ptr> align 1 [[TMP1]], <4 x i1> splat (i1 true)), !alias.scope [[META31:![0-9]+]], !noalias [[META28]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP1]], i32 2
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP5]], <4 x ptr> align 1 [[TMP7]], <4 x i1> splat (i1 true)), !alias.scope [[META31]], !noalias [[META28]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP6]], <4 x ptr> align 1 [[TMP8]], <4 x i1> splat (i1 true)), !alias.scope [[META31]], !noalias [[META28]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ], [ [[X]], %[[ENTRY]] ]
+; CHECK-NEXT: [[POINTER_PHI2:%.*]] = phi ptr [ [[PTR_IND6:%.*]], %[[VECTOR_BODY]] ], [ [[Z]], %[[ENTRY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI2]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT: [[VECTOR_GEP3:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, <4 x ptr> [[VECTOR_GEP3]], i32 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[VECTOR_GEP3]], <4 x i1> splat (i1 true), <4 x i8> poison), !alias.scope [[META28:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, <4 x ptr> [[VECTOR_GEP3]], i32 2
+; CHECK-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP0]], <4 x i1> splat (i1 true), <4 x i8> poison), !alias.scope [[META28]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP1]], <4 x i1> splat (i1 true), <4 x i8> poison), !alias.scope [[META28]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], splat (i8 10)
+; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER4]]
+; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER5]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, <4 x ptr> [[VECTOR_GEP]], i32 1
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP2]], <4 x ptr> align 1 [[VECTOR_GEP]], <4 x i1> splat (i1 true)), !alias.scope [[META31:![0-9]+]], !noalias [[META28]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, <4 x ptr> [[VECTOR_GEP]], i32 2
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP3]], <4 x ptr> align 1 [[TMP5]], <4 x i1> splat (i1 true)), !alias.scope [[META31]], !noalias [[META28]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP4]], <4 x ptr> align 1 [[TMP6]], <4 x i1> splat (i1 true)), !alias.scope [[META31]], !noalias [[META28]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 12
-; CHECK-NEXT: [[PTR_IND6]] = getelementptr i8, ptr [[POINTER_PHI5]], i32 12
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP9]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK: for.body:
-; CHECK-NEXT: [[X_ADDR_050:%.*]] = phi ptr [ [[INCDEC_PTR2:%.*]], [[FOR_BODY]] ], [ [[X]], [[ENTRY]] ]
-; CHECK-NEXT: [[Z_ADDR_049:%.*]] = phi ptr [ [[INCDEC_PTR34:%.*]], [[FOR_BODY]] ], [ [[Z]], [[ENTRY]] ]
-; CHECK-NEXT: [[I_048:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT: [[PTR_IND6]] = getelementptr i8, ptr [[POINTER_PHI2]], i32 12
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP7]], label %[[END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[X_ADDR_050:%.*]] = phi ptr [ [[INCDEC_PTR2:%.*]], %[[FOR_BODY]] ], [ [[X]], %[[ENTRY]] ]
+; CHECK-NEXT: [[Z_ADDR_049:%.*]] = phi ptr [ [[INCDEC_PTR34:%.*]], %[[FOR_BODY]] ], [ [[Z]], %[[ENTRY]] ]
+; CHECK-NEXT: [[I_048:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[X_ADDR_050]], i32 1
-; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[X_ADDR_050]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[X_ADDR_050]], align 1
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[X_ADDR_050]], i32 2
-; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
; CHECK-NEXT: [[INCDEC_PTR2]] = getelementptr inbounds nuw i8, ptr [[X_ADDR_050]], i32 3
-; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[INCDEC_PTR1]], align 1
-; CHECK-NEXT: [[MUL:%.*]] = mul i8 [[TMP10]], 10
-; CHECK-NEXT: [[MUL1:%.*]] = mul i8 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[MUL2:%.*]] = mul i8 [[TMP10]], [[TMP12]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[INCDEC_PTR1]], align 1
+; CHECK-NEXT: [[MUL:%.*]] = mul i8 [[TMP8]], 10
+; CHECK-NEXT: [[MUL1:%.*]] = mul i8 [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[MUL2:%.*]] = mul i8 [[TMP8]], [[TMP10]]
; CHECK-NEXT: [[INCDEC_PTR32:%.*]] = getelementptr inbounds nuw i8, ptr [[Z_ADDR_049]], i32 1
; CHECK-NEXT: store i8 [[MUL]], ptr [[Z_ADDR_049]], align 1
; CHECK-NEXT: [[INCDEC_PTR33:%.*]] = getelementptr inbounds nuw i8, ptr [[Z_ADDR_049]], i32 2
@@ -924,8 +942,8 @@ define hidden void @mult_ptr_iv(ptr noalias nocapture readonly %x, ptr noalias n
; CHECK-NEXT: store i8 [[MUL2]], ptr [[INCDEC_PTR33]], align 1
; CHECK-NEXT: [[INC]] = add nuw i32 [[I_048]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END]], label %[[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll b/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll
index a16b2a65654f3..72813c8693468 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll
@@ -1,44 +1,44 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-apple-macosx -passes=loop-vectorize,simplifycfg -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s
define void @test_pr55096(i64 %c, ptr %p) {
; CHECK-LABEL: define void @test_pr55096(
; CHECK-SAME: i64 [[C:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[C]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 122, i64 123>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 122, i64 123>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[INDEX]] to i16
; CHECK-NEXT: [[TMP0:%.*]] = mul i16 [[DOTCAST]], 2008
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i16 6229, [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp uge <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; CHECK: pred.store.if:
-; CHECK-NEXT: [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 2008
-; CHECK-NEXT: [[TMP6:%.*]] = udiv i16 4943, [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[P]], i16 [[TMP6]]
-; CHECK-NEXT: store i16 0, ptr [[TMP7]], align 2
-; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
-; CHECK: pred.store.continue:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
-; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; CHECK: pred.store.if1:
-; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[OFFSET_IDX]], 2008
-; CHECK-NEXT: [[TMP11:%.*]] = add i16 [[TMP10]], 2008
-; CHECK-NEXT: [[TMP12:%.*]] = udiv i16 4943, [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[P]], i16 [[TMP12]]
-; CHECK-NEXT: store i16 0, ptr [[TMP13]], align 2
-; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; CHECK: pred.store.continue2:
+; CHECK-NEXT: [[TMP1:%.*]] = icmp uge <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], 2008
+; CHECK-NEXT: [[TMP4:%.*]] = udiv i16 4943, [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P]], i16 [[TMP4]]
+; CHECK-NEXT: store i16 0, ptr [[TMP5]], align 2
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[OFFSET_IDX]], 2008
+; CHECK-NEXT: [[TMP8:%.*]] = add i16 [[TMP7]], 2008
+; CHECK-NEXT: [[TMP9:%.*]] = udiv i16 4943, [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[P]], i16 [[TMP9]]
+; CHECK-NEXT: store i16 0, ptr [[TMP10]], align 2
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 340
-; CHECK-NEXT: br i1 [[TMP15]], label [[EXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: exit:
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 340
+; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
index 28debf57d69e8..899a10f127955 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -mcpu=skx -S -passes=loop-vectorize,instcombine,simplifycfg -simplifycfg-require-and-preserve-domtree=1 -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses < %s | FileCheck %s -check-prefix=DISABLED_MASKED_STRIDED
; RUN: opt -mcpu=skx -S -passes=loop-vectorize,instcombine,simplifycfg -simplifycfg-require-and-preserve-domtree=1 -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses < %s | FileCheck %s -check-prefix=ENABLED_MASKED_STRIDED
@@ -30,143 +30,145 @@ target triple = "i386-unknown-linux-gnu"
; }
define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) {
-; DISABLED_MASKED_STRIDED-LABEL: @masked_strided1(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; DISABLED_MASKED_STRIDED-LABEL: define void @masked_strided1(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE14]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp samesign ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.load.continue2:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE2]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], %[[PRED_LOAD_IF7]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], [[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], %[[PRED_LOAD_IF9]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], [[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], %[[PRED_LOAD_IF11]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP49]], ptr align 1 [[TMP50]], <8 x i1> [[TMP0]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @masked_strided1(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-LABEL: define void @masked_strided1(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp samesign ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP3]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1016
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[IX_09:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ 1016, [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[IX_09:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_INC:.*]] ], [ 1016, %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_09]], [[CONV]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; ENABLED_MASKED_STRIDED: if.then:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; ENABLED_MASKED_STRIDED: [[IF_THEN]]:
; ENABLED_MASKED_STRIDED-NEXT: [[MUL:%.*]] = shl nuw nsw i32 [[IX_09]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[MUL]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[IX_09]]
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP5]], ptr [[ARRAYIDX3]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[FOR_INC]]
-; ENABLED_MASKED_STRIDED: for.inc:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_INC]]
+; ENABLED_MASKED_STRIDED: [[FOR_INC]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INC]] = add nuw nsw i32 [[IX_09]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -203,129 +205,131 @@ for.end:
; And-ed with the gaps mask.
;
define void @masked_strided1_optsize(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) optsize {
-; DISABLED_MASKED_STRIDED-LABEL: @masked_strided1_optsize(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; DISABLED_MASKED_STRIDED-LABEL: define void @masked_strided1_optsize(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR1:[0-9]+]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE14]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp samesign ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.load.continue2:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE2]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], %[[PRED_LOAD_IF7]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], [[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], %[[PRED_LOAD_IF9]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], [[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], %[[PRED_LOAD_IF11]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP49]], ptr align 1 [[TMP50]], <8 x i1> [[TMP0]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @masked_strided1_optsize(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-LABEL: define void @masked_strided1_optsize(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR1:[0-9]+]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp samesign ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[TMP3]], <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP4]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -382,12 +386,13 @@ for.end:
; }
;
define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard, i32 %n) optsize {
-; DISABLED_MASKED_STRIDED-LABEL: @masked_strided1_optsize_unknown_tc(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; DISABLED_MASKED_STRIDED: vector.ph:
-; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; DISABLED_MASKED_STRIDED-LABEL: define void @masked_strided1_optsize_unknown_tc(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; DISABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
@@ -395,109 +400,110 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE16:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE16]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP3]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP5]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> poison, i8 [[TMP7]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP3]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP11]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP12]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[TMP13]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF3]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP3]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP17]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[TMP19]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], [[PRED_LOAD_IF5]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP3]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], %[[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP23]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = insertelement <8 x i8> [[TMP21]], i8 [[TMP25]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], [[PRED_LOAD_IF7]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP3]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], %[[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP29]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = insertelement <8 x i8> [[TMP27]], i8 [[TMP31]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], [[PRED_LOAD_IF9]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP3]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], %[[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP35]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = insertelement <8 x i8> [[TMP33]], i8 [[TMP37]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], [[PRED_LOAD_IF11]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP3]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], %[[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP41]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[TMP43]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], [[PRED_LOAD_IF13]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP3]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.if15:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], %[[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP47]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = insertelement <8 x i8> [[TMP45]], i8 [[TMP49]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.continue16:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], [[PRED_LOAD_IF15]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
-; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP3]])
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @masked_strided1_optsize_unknown_tc(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; ENABLED_MASKED_STRIDED: vector.ph:
-; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-LABEL: define void @masked_strided1_optsize_unknown_tc(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; ENABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
@@ -505,26 +511,26 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 [[TMP2]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl i32 [[INDEX]], 1
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
-; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP3]], <16 x i1> [[TMP5]], <16 x i8> poison)
+; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[TMP5]], <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP6]], <8 x i1> [[TMP4]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP6]], <8 x i1> [[TMP2]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -586,12 +592,13 @@ for.end:
; }
;
define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard, i32 %n) optsize {
-; DISABLED_MASKED_STRIDED-LABEL: @masked_strided3_optsize_unknown_tc(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; DISABLED_MASKED_STRIDED: vector.ph:
-; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; DISABLED_MASKED_STRIDED-LABEL: define void @masked_strided3_optsize_unknown_tc(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; DISABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
@@ -599,109 +606,110 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE16:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE16]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = mul nsw <8 x i32> [[VEC_IND]], splat (i32 3)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP3]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = mul nsw <8 x i32> [[VEC_IND]], splat (i32 3)
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP5]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> poison, i8 [[TMP7]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP3]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP11]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP12]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[TMP13]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF3]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP3]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP17]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[TMP19]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], [[PRED_LOAD_IF5]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP3]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], %[[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP23]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = insertelement <8 x i8> [[TMP21]], i8 [[TMP25]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], [[PRED_LOAD_IF7]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP3]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], %[[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP29]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = insertelement <8 x i8> [[TMP27]], i8 [[TMP31]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], [[PRED_LOAD_IF9]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP3]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], %[[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP35]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = insertelement <8 x i8> [[TMP33]], i8 [[TMP37]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], [[PRED_LOAD_IF11]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP3]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], %[[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP41]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[TMP43]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], [[PRED_LOAD_IF13]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP3]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.if15:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], %[[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP47]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = insertelement <8 x i8> [[TMP45]], i8 [[TMP49]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.continue16:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], [[PRED_LOAD_IF15]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
-; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP3]])
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @masked_strided3_optsize_unknown_tc(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; ENABLED_MASKED_STRIDED: vector.ph:
-; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-LABEL: define void @masked_strided3_optsize_unknown_tc(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; ENABLED_MASKED_STRIDED-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
@@ -709,101 +717,101 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE16:%.*]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE16]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = mul nsw <8 x i32> [[VEC_IND]], splat (i32 3)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP5]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = load i8, ptr [[TMP52]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> poison, i8 [[TMP53]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; ENABLED_MASKED_STRIDED: pred.load.continue:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP4]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if3:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP5]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> poison, i8 [[TMP7]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP11]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP12]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[TMP13]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; ENABLED_MASKED_STRIDED: pred.load.continue4:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF3]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP4]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if5:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF3]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
; ENABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP17]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[TMP19]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; ENABLED_MASKED_STRIDED: pred.load.continue6:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], [[PRED_LOAD_IF5]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP4]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if7:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], %[[PRED_LOAD_IF5]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP23]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = insertelement <8 x i8> [[TMP21]], i8 [[TMP25]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; ENABLED_MASKED_STRIDED: pred.load.continue8:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], [[PRED_LOAD_IF7]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP4]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if9:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], %[[PRED_LOAD_IF7]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
; ENABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP29]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = insertelement <8 x i8> [[TMP27]], i8 [[TMP31]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; ENABLED_MASKED_STRIDED: pred.load.continue10:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], [[PRED_LOAD_IF9]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP4]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if11:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], %[[PRED_LOAD_IF9]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
; ENABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP35]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = insertelement <8 x i8> [[TMP33]], i8 [[TMP37]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; ENABLED_MASKED_STRIDED: pred.load.continue12:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], [[PRED_LOAD_IF11]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP4]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if13:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], %[[PRED_LOAD_IF11]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
; ENABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP41]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[TMP43]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; ENABLED_MASKED_STRIDED: pred.load.continue14:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], [[PRED_LOAD_IF13]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP4]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16]]
-; ENABLED_MASKED_STRIDED: pred.load.if15:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], %[[PRED_LOAD_IF13]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
; ENABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP47]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = insertelement <8 x i8> [[TMP45]], i8 [[TMP49]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; ENABLED_MASKED_STRIDED: pred.load.continue16:
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = phi <8 x i8> [ [[TMP45]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], [[PRED_LOAD_IF15]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP6]], <8 x i1> [[TMP4]])
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -861,36 +869,37 @@ for.end:
; }
;
define void @unconditional_strided1_optsize(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) optsize {
-; DISABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; DISABLED_MASKED_STRIDED-LABEL: define void @unconditional_strided1_optsize(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR1]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = extractelement <8 x i32> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP1]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP5]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP7]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP11]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP13]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP2]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = load i8, ptr [[TMP4]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP6]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP8]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = load i8, ptr [[TMP10]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP12]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP14]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[TMP0]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP0]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP0]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP0]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP0]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP0]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP0]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP1]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP2]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP6]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP7]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP9]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = load i8, ptr [[TMP10]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP11]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP12]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = load i8, ptr [[TMP13]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP14]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP15]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = load i8, ptr [[TMP16]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = insertelement <8 x i8> poison, i8 [[TMP17]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP18]], i64 1
@@ -900,30 +909,31 @@ define void @unconditional_strided1_optsize(ptr noalias nocapture readonly %p, p
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP29]], i8 [[TMP22]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = insertelement <8 x i8> [[TMP30]], i8 [[TMP23]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP24]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: store <8 x i8> [[TMP32]], ptr [[TMP33]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-LABEL: define void @unconditional_strided1_optsize(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR1]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = shl nuw nsw i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP0]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP0]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>, <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: store <8 x i8> [[STRIDED_VEC]], ptr [[TMP2]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -966,140 +976,142 @@ for.end:
; }
;
define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %n) optsize {
-; DISABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize_unknown_tc(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; DISABLED_MASKED_STRIDED: vector.ph:
+; DISABLED_MASKED_STRIDED-LABEL: define void @unconditional_strided1_optsize_unknown_tc(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; DISABLED_MASKED_STRIDED-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP6]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE14]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.load.continue2:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE2]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], %[[PRED_LOAD_IF7]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], [[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], %[[PRED_LOAD_IF9]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], [[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], %[[PRED_LOAD_IF11]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP49]], ptr align 1 [[TMP50]], <8 x i1> [[TMP0]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize_unknown_tc(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; ENABLED_MASKED_STRIDED: vector.ph:
+; ENABLED_MASKED_STRIDED-LABEL: define void @unconditional_strided1_optsize_unknown_tc(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; ENABLED_MASKED_STRIDED-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N]], 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP6]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[INDEX]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IV:%.*]] = or disjoint <8 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[TMP3]], <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP4]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -1153,349 +1165,351 @@ for.end:
;}
;
define void @masked_strided2(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) {
-; DISABLED_MASKED_STRIDED-LABEL: @masked_strided2(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; DISABLED_MASKED_STRIDED-LABEL: define void @masked_strided2(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE60:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE60]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE60:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp samesign ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.load.continue2:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE2]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], %[[PRED_LOAD_IF7]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], [[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], %[[PRED_LOAD_IF9]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], [[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], %[[PRED_LOAD_IF11]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = or disjoint <8 x i32> [[TMP1]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if15:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.continue16:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], [[PRED_LOAD_IF15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], %[[PRED_LOAD_IF15]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if17:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF17]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE18]]
-; DISABLED_MASKED_STRIDED: pred.load.continue18:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], [[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], [[PRED_LOAD_IF17]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE18]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE18]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], %[[PRED_LOAD_IF17]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if19:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF19]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE20]]
-; DISABLED_MASKED_STRIDED: pred.load.continue20:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], [[PRED_LOAD_IF19]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE20]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE20]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], %[[PRED_LOAD_IF19]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if21:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF21]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE22]]
-; DISABLED_MASKED_STRIDED: pred.load.continue22:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], [[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], [[PRED_LOAD_IF21]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE22]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE22]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], %[[PRED_LOAD_IF21]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if23:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF23]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE24]]
-; DISABLED_MASKED_STRIDED: pred.load.continue24:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], [[PRED_LOAD_IF23]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE24]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE24]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], %[[PRED_LOAD_IF23]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if25:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF25]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE26]]
-; DISABLED_MASKED_STRIDED: pred.load.continue26:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], [[PRED_LOAD_IF25]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE26]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE26]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], %[[PRED_LOAD_IF25]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if27:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF27]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE28]]
-; DISABLED_MASKED_STRIDED: pred.load.continue28:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], [[PRED_LOAD_IF27]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE28]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE28]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], %[[PRED_LOAD_IF27]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if29:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF29]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE30]]
-; DISABLED_MASKED_STRIDED: pred.load.continue30:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], [[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], [[PRED_LOAD_IF29]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE30]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE30]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], %[[PRED_LOAD_IF29]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[TMP49]], <8 x i8> [[TMP98]])
; DISABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP101]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP103]], ptr [[TMP102]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.store.continue:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if31:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF31]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP107]], ptr [[TMP106]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE32]]
-; DISABLED_MASKED_STRIDED: pred.store.continue32:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE32]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if33:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF33]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], ptr [[TMP110]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]]
-; DISABLED_MASKED_STRIDED: pred.store.continue34:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE34]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE34]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if35:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF35:.*]], label %[[PRED_STORE_CONTINUE36:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF35]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP115]], ptr [[TMP114]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]]
-; DISABLED_MASKED_STRIDED: pred.store.continue36:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE36]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE36]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if37:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF37:.*]], label %[[PRED_STORE_CONTINUE38:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF37]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], ptr [[TMP118]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]]
-; DISABLED_MASKED_STRIDED: pred.store.continue38:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE38]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE38]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if39:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF39:.*]], label %[[PRED_STORE_CONTINUE40:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF39]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP123]], ptr [[TMP122]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]]
-; DISABLED_MASKED_STRIDED: pred.store.continue40:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE40]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE40]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if41:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF41:.*]], label %[[PRED_STORE_CONTINUE42:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF41]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], ptr [[TMP126]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]]
-; DISABLED_MASKED_STRIDED: pred.store.continue42:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE42]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE42]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if43:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF43:.*]], label %[[PRED_STORE_CONTINUE44:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF43]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP131]], ptr [[TMP130]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]]
-; DISABLED_MASKED_STRIDED: pred.store.continue44:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE44]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE44]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = sub <8 x i8> zeroinitializer, [[TMP99]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if45:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP136]], ptr [[TMP135]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]]
-; DISABLED_MASKED_STRIDED: pred.store.continue46:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if47:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP140]], ptr [[TMP139]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]]
-; DISABLED_MASKED_STRIDED: pred.store.continue48:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if49:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF45:.*]], label %[[PRED_STORE_CONTINUE46:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF45]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP133]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], ptr [[TMP134]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE46]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE46]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF47:.*]], label %[[PRED_STORE_CONTINUE48:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF47]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP136]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], ptr [[TMP137]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE48]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE48]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF49:.*]], label %[[PRED_STORE_CONTINUE50:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF49]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP139]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], ptr [[TMP140]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE50]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE50]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF51:.*]], label %[[PRED_STORE_CONTINUE52:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF51]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], ptr [[TMP143]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]]
-; DISABLED_MASKED_STRIDED: pred.store.continue50:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if51:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP148]], ptr [[TMP147]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]]
-; DISABLED_MASKED_STRIDED: pred.store.continue52:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if53:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], ptr [[TMP151]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]]
-; DISABLED_MASKED_STRIDED: pred.store.continue54:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if55:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE52]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE52]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF53:.*]], label %[[PRED_STORE_CONTINUE54:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF53]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP145]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP147]], ptr [[TMP146]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE54]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE54]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF55:.*]], label %[[PRED_STORE_CONTINUE56:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF55]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP148]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP150]], ptr [[TMP149]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE56]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE56]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF57:.*]], label %[[PRED_STORE_CONTINUE58:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF57]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP151]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP153]], ptr [[TMP152]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE58]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE58]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF59:.*]], label %[[PRED_STORE_CONTINUE60]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF59]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], ptr [[TMP155]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]]
-; DISABLED_MASKED_STRIDED: pred.store.continue56:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if57:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], ptr [[TMP159]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]]
-; DISABLED_MASKED_STRIDED: pred.store.continue58:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
-; DISABLED_MASKED_STRIDED: pred.store.if59:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP164]], ptr [[TMP163]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]]
-; DISABLED_MASKED_STRIDED: pred.store.continue60:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE60]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE60]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @masked_strided2(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-LABEL: define void @masked_strided2(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp samesign ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8> [[STRIDED_VEC1]])
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = sub <8 x i8> zeroinitializer, [[TMP4]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8> [[STRIDED_VEC1]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[Q]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = sub <8 x i8> zeroinitializer, [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP6]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -1550,638 +1564,640 @@ for.end:
;}
;
define void @masked_strided2_reverse(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) {
-; DISABLED_MASKED_STRIDED-LABEL: @masked_strided2_reverse(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; DISABLED_MASKED_STRIDED-LABEL: define void @masked_strided2_reverse(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE60:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 1024, i32 1023, i32 1022, i32 1021, i32 1020, i32 1019, i32 1018, i32 1017>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE60]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE60:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 1024, i32 1023, i32 1022, i32 1021, i32 1020, i32 1019, i32 1018, i32 1017>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.load.continue2:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE2]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], %[[PRED_LOAD_IF7]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], [[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], %[[PRED_LOAD_IF9]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], [[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], %[[PRED_LOAD_IF11]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = or disjoint <8 x i32> [[TMP1]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if15:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.continue16:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], [[PRED_LOAD_IF15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], %[[PRED_LOAD_IF15]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if17:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF17]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE18]]
-; DISABLED_MASKED_STRIDED: pred.load.continue18:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], [[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], [[PRED_LOAD_IF17]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE18]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE18]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], %[[PRED_LOAD_IF17]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if19:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF19]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE20]]
-; DISABLED_MASKED_STRIDED: pred.load.continue20:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], [[PRED_LOAD_IF19]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE20]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE20]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], %[[PRED_LOAD_IF19]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if21:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF21]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE22]]
-; DISABLED_MASKED_STRIDED: pred.load.continue22:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], [[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], [[PRED_LOAD_IF21]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE22]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE22]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], %[[PRED_LOAD_IF21]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if23:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF23]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE24]]
-; DISABLED_MASKED_STRIDED: pred.load.continue24:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], [[PRED_LOAD_IF23]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE24]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE24]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], %[[PRED_LOAD_IF23]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if25:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF25]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE26]]
-; DISABLED_MASKED_STRIDED: pred.load.continue26:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], [[PRED_LOAD_IF25]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE26]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE26]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], %[[PRED_LOAD_IF25]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if27:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF27]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE28]]
-; DISABLED_MASKED_STRIDED: pred.load.continue28:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], [[PRED_LOAD_IF27]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE28]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE28]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], %[[PRED_LOAD_IF27]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if29:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF29]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE30]]
-; DISABLED_MASKED_STRIDED: pred.load.continue30:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], [[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], [[PRED_LOAD_IF29]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE30]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE30]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], %[[PRED_LOAD_IF29]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[TMP49]], <8 x i8> [[TMP98]])
; DISABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP101]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP103]], ptr [[TMP102]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.store.continue:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if31:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF31]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP107]], ptr [[TMP106]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE32]]
-; DISABLED_MASKED_STRIDED: pred.store.continue32:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE32]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if33:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF33]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], ptr [[TMP110]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]]
-; DISABLED_MASKED_STRIDED: pred.store.continue34:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE34]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE34]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if35:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF35:.*]], label %[[PRED_STORE_CONTINUE36:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF35]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP115]], ptr [[TMP114]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]]
-; DISABLED_MASKED_STRIDED: pred.store.continue36:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE36]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE36]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if37:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF37:.*]], label %[[PRED_STORE_CONTINUE38:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF37]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], ptr [[TMP118]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]]
-; DISABLED_MASKED_STRIDED: pred.store.continue38:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE38]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE38]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if39:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF39:.*]], label %[[PRED_STORE_CONTINUE40:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF39]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP123]], ptr [[TMP122]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]]
-; DISABLED_MASKED_STRIDED: pred.store.continue40:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE40]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE40]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if41:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF41:.*]], label %[[PRED_STORE_CONTINUE42:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF41]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], ptr [[TMP126]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]]
-; DISABLED_MASKED_STRIDED: pred.store.continue42:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE42]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE42]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if43:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF43:.*]], label %[[PRED_STORE_CONTINUE44:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF43]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP131]], ptr [[TMP130]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]]
-; DISABLED_MASKED_STRIDED: pred.store.continue44:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE44]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE44]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = sub <8 x i8> zeroinitializer, [[TMP99]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if45:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP136]], ptr [[TMP135]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]]
-; DISABLED_MASKED_STRIDED: pred.store.continue46:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if47:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP140]], ptr [[TMP139]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]]
-; DISABLED_MASKED_STRIDED: pred.store.continue48:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if49:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF45:.*]], label %[[PRED_STORE_CONTINUE46:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF45]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP133]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], ptr [[TMP134]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE46]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE46]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF47:.*]], label %[[PRED_STORE_CONTINUE48:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF47]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP136]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], ptr [[TMP137]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE48]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE48]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF49:.*]], label %[[PRED_STORE_CONTINUE50:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF49]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP139]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], ptr [[TMP140]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE50]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE50]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF51:.*]], label %[[PRED_STORE_CONTINUE52:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF51]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], ptr [[TMP143]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]]
-; DISABLED_MASKED_STRIDED: pred.store.continue50:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if51:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP148]], ptr [[TMP147]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]]
-; DISABLED_MASKED_STRIDED: pred.store.continue52:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if53:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], ptr [[TMP151]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]]
-; DISABLED_MASKED_STRIDED: pred.store.continue54:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if55:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE52]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE52]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF53:.*]], label %[[PRED_STORE_CONTINUE54:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF53]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP145]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP147]], ptr [[TMP146]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE54]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE54]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF55:.*]], label %[[PRED_STORE_CONTINUE56:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF55]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP148]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP150]], ptr [[TMP149]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE56]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE56]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF57:.*]], label %[[PRED_STORE_CONTINUE58:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF57]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP151]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP153]], ptr [[TMP152]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE58]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE58]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF59:.*]], label %[[PRED_STORE_CONTINUE60]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF59]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], ptr [[TMP155]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]]
-; DISABLED_MASKED_STRIDED: pred.store.continue56:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if57:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], ptr [[TMP159]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]]
-; DISABLED_MASKED_STRIDED: pred.store.continue58:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
-; DISABLED_MASKED_STRIDED: pred.store.if59:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP164]], ptr [[TMP163]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]]
-; DISABLED_MASKED_STRIDED: pred.store.continue60:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE60]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE60]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 -8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @masked_strided2_reverse(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-LABEL: define void @masked_strided2_reverse(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE60:%.*]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 1024, i32 1023, i32 1022, i32 1021, i32 1020, i32 1019, i32 1018, i32 1017>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE60]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE60:.*]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 1024, i32 1023, i32 1022, i32 1021, i32 1020, i32 1019, i32 1018, i32 1017>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; ENABLED_MASKED_STRIDED: pred.load.continue:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if1:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF1]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; ENABLED_MASKED_STRIDED: pred.load.continue2:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE2]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if3:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; ENABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; ENABLED_MASKED_STRIDED: pred.load.continue4:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if5:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; ENABLED_MASKED_STRIDED: pred.load.continue6:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if7:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; ENABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; ENABLED_MASKED_STRIDED: pred.load.continue8:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], %[[PRED_LOAD_IF7]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if9:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; ENABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; ENABLED_MASKED_STRIDED: pred.load.continue10:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], [[PRED_LOAD_IF9]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], %[[PRED_LOAD_IF9]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if11:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; ENABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; ENABLED_MASKED_STRIDED: pred.load.continue12:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], [[PRED_LOAD_IF11]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], %[[PRED_LOAD_IF11]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if13:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; ENABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; ENABLED_MASKED_STRIDED: pred.load.continue14:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = or disjoint <8 x i32> [[TMP1]], splat (i32 1)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if15:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; ENABLED_MASKED_STRIDED: pred.load.continue16:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], [[PRED_LOAD_IF15]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], %[[PRED_LOAD_IF15]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if17:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF17]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE18]]
-; ENABLED_MASKED_STRIDED: pred.load.continue18:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], [[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], [[PRED_LOAD_IF17]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE18]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE18]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], %[[PRED_LOAD_IF17]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if19:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF19]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
; ENABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE20]]
-; ENABLED_MASKED_STRIDED: pred.load.continue20:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], [[PRED_LOAD_IF19]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE20]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE20]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], %[[PRED_LOAD_IF19]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if21:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF21]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE22]]
-; ENABLED_MASKED_STRIDED: pred.load.continue22:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], [[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], [[PRED_LOAD_IF21]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE22]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE22]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], %[[PRED_LOAD_IF21]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if23:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF23]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
; ENABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE24]]
-; ENABLED_MASKED_STRIDED: pred.load.continue24:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], [[PRED_LOAD_IF23]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE24]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE24]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], %[[PRED_LOAD_IF23]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if25:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF25]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
; ENABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE26]]
-; ENABLED_MASKED_STRIDED: pred.load.continue26:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], [[PRED_LOAD_IF25]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE26]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE26]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], %[[PRED_LOAD_IF25]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if27:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF27]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
; ENABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE28]]
-; ENABLED_MASKED_STRIDED: pred.load.continue28:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], [[PRED_LOAD_IF27]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE28]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE28]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], %[[PRED_LOAD_IF27]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
-; ENABLED_MASKED_STRIDED: pred.load.if29:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_IF29]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; ENABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE30]]
-; ENABLED_MASKED_STRIDED: pred.load.continue30:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], [[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], [[PRED_LOAD_IF29]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE30]]
+; ENABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE30]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], %[[PRED_LOAD_IF29]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[TMP49]], <8 x i8> [[TMP98]])
; ENABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP101]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP103]], ptr [[TMP102]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; ENABLED_MASKED_STRIDED: pred.store.continue:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if31:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF31]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP107]], ptr [[TMP106]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE32]]
-; ENABLED_MASKED_STRIDED: pred.store.continue32:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE32]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if33:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF33]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; ENABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], ptr [[TMP110]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]]
-; ENABLED_MASKED_STRIDED: pred.store.continue34:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE34]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE34]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if35:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF35:.*]], label %[[PRED_STORE_CONTINUE36:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF35]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP115]], ptr [[TMP114]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]]
-; ENABLED_MASKED_STRIDED: pred.store.continue36:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE36]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE36]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if37:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF37:.*]], label %[[PRED_STORE_CONTINUE38:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF37]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; ENABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], ptr [[TMP118]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]]
-; ENABLED_MASKED_STRIDED: pred.store.continue38:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE38]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE38]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if39:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF39:.*]], label %[[PRED_STORE_CONTINUE40:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF39]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; ENABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP123]], ptr [[TMP122]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]]
-; ENABLED_MASKED_STRIDED: pred.store.continue40:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE40]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE40]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if41:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF41:.*]], label %[[PRED_STORE_CONTINUE42:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF41]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; ENABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], ptr [[TMP126]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]]
-; ENABLED_MASKED_STRIDED: pred.store.continue42:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE42]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE42]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if43:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF43:.*]], label %[[PRED_STORE_CONTINUE44:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF43]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; ENABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP131]], ptr [[TMP130]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]]
-; ENABLED_MASKED_STRIDED: pred.store.continue44:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE44]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE44]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = sub <8 x i8> zeroinitializer, [[TMP99]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if45:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP136]], ptr [[TMP135]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]]
-; ENABLED_MASKED_STRIDED: pred.store.continue46:
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if47:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP140]], ptr [[TMP139]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]]
-; ENABLED_MASKED_STRIDED: pred.store.continue48:
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if49:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF45:.*]], label %[[PRED_STORE_CONTINUE46:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF45]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP133]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], ptr [[TMP134]], align 1
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE46]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE46]]:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF47:.*]], label %[[PRED_STORE_CONTINUE48:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF47]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP136]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
+; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], ptr [[TMP137]], align 1
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE48]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE48]]:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF49:.*]], label %[[PRED_STORE_CONTINUE50:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF49]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP139]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], ptr [[TMP140]], align 1
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE50]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE50]]:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF51:.*]], label %[[PRED_STORE_CONTINUE52:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF51]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], ptr [[TMP143]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]]
-; ENABLED_MASKED_STRIDED: pred.store.continue50:
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if51:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP148]], ptr [[TMP147]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]]
-; ENABLED_MASKED_STRIDED: pred.store.continue52:
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if53:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], ptr [[TMP151]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]]
-; ENABLED_MASKED_STRIDED: pred.store.continue54:
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if55:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE52]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE52]]:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF53:.*]], label %[[PRED_STORE_CONTINUE54:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF53]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP145]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
+; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP147]], ptr [[TMP146]], align 1
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE54]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE54]]:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF55:.*]], label %[[PRED_STORE_CONTINUE56:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF55]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP148]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP150]], ptr [[TMP149]], align 1
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE56]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE56]]:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF57:.*]], label %[[PRED_STORE_CONTINUE58:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF57]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP151]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
+; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP153]], ptr [[TMP152]], align 1
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE58]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE58]]:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF59:.*]], label %[[PRED_STORE_CONTINUE60]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF59]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; ENABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], ptr [[TMP155]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]]
-; ENABLED_MASKED_STRIDED: pred.store.continue56:
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if57:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], ptr [[TMP159]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]]
-; ENABLED_MASKED_STRIDED: pred.store.continue58:
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
-; ENABLED_MASKED_STRIDED: pred.store.if59:
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP164]], ptr [[TMP163]], align 1
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]]
-; ENABLED_MASKED_STRIDED: pred.store.continue60:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE60]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE60]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 -8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -2245,367 +2261,369 @@ for.end:
;}
;
define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %guard, i32 %n) optsize {
-; DISABLED_MASKED_STRIDED-LABEL: @masked_strided2_unknown_tc(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CMP22:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP22]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; DISABLED_MASKED_STRIDED: vector.ph:
+; DISABLED_MASKED_STRIDED-LABEL: define void @masked_strided2_unknown_tc(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[GUARD:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; DISABLED_MASKED_STRIDED-NEXT: [[CMP22:%.*]] = icmp sgt i32 [[N]], 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP22]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD:%.*]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE62:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE62]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE62:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE62]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP3]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP5]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> poison, i8 [[TMP7]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP3]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP11]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP12]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[TMP13]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF3]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP3]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = phi <8 x i8> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP17]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[TMP19]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], [[PRED_LOAD_IF5]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP3]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = phi <8 x i8> [ [[TMP15]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], %[[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP23]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = insertelement <8 x i8> [[TMP21]], i8 [[TMP25]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], [[PRED_LOAD_IF7]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP3]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = phi <8 x i8> [ [[TMP21]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP26]], %[[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP29]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = insertelement <8 x i8> [[TMP27]], i8 [[TMP31]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], [[PRED_LOAD_IF9]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP3]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = phi <8 x i8> [ [[TMP27]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP32]], %[[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP34]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP35]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = insertelement <8 x i8> [[TMP33]], i8 [[TMP37]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], [[PRED_LOAD_IF11]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP3]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = phi <8 x i8> [ [[TMP33]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP38]], %[[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP40]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP41]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[TMP43]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], [[PRED_LOAD_IF13]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP3]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if15:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = phi <8 x i8> [ [[TMP39]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP44]], %[[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP46]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP47]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = insertelement <8 x i8> [[TMP45]], i8 [[TMP49]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.continue16:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], [[PRED_LOAD_IF15]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = or disjoint <8 x i32> [[TMP2]], splat (i32 1)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = extractelement <8 x i1> [[TMP3]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if17:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = or disjoint <8 x i32> [[TMP3]], splat (i32 1)
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF17]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = extractelement <8 x i32> [[TMP52]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP54]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = load i8, ptr [[TMP55]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = insertelement <8 x i8> poison, i8 [[TMP56]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE18]]
-; DISABLED_MASKED_STRIDED: pred.load.continue18:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE16]] ], [ [[TMP57]], [[PRED_LOAD_IF17]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = extractelement <8 x i1> [[TMP3]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP59]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if19:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE18]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE18]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = phi <8 x i8> [ poison, %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP57]], %[[PRED_LOAD_IF17]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP59]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF19]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = extractelement <8 x i32> [[TMP52]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP60]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = load i8, ptr [[TMP61]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = insertelement <8 x i8> [[TMP58]], i8 [[TMP62]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE20]]
-; DISABLED_MASKED_STRIDED: pred.load.continue20:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = phi <8 x i8> [ [[TMP58]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP63]], [[PRED_LOAD_IF19]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = extractelement <8 x i1> [[TMP3]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP65]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if21:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE20]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE20]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = phi <8 x i8> [ [[TMP58]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP63]], %[[PRED_LOAD_IF19]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP65]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF21]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = extractelement <8 x i32> [[TMP52]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP66]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = load i8, ptr [[TMP67]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = insertelement <8 x i8> [[TMP64]], i8 [[TMP68]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE22]]
-; DISABLED_MASKED_STRIDED: pred.load.continue22:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = phi <8 x i8> [ [[TMP64]], [[PRED_LOAD_CONTINUE20]] ], [ [[TMP69]], [[PRED_LOAD_IF21]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = extractelement <8 x i1> [[TMP3]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP71]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if23:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE22]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE22]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = phi <8 x i8> [ [[TMP64]], %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP69]], %[[PRED_LOAD_IF21]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP71]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF23]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = extractelement <8 x i32> [[TMP52]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP72]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = load i8, ptr [[TMP73]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = insertelement <8 x i8> [[TMP70]], i8 [[TMP74]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE24]]
-; DISABLED_MASKED_STRIDED: pred.load.continue24:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = phi <8 x i8> [ [[TMP70]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP75]], [[PRED_LOAD_IF23]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = extractelement <8 x i1> [[TMP3]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP77]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if25:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE24]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE24]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = phi <8 x i8> [ [[TMP70]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP75]], %[[PRED_LOAD_IF23]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP77]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF25]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = extractelement <8 x i32> [[TMP52]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP78]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = load i8, ptr [[TMP79]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = insertelement <8 x i8> [[TMP76]], i8 [[TMP80]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE26]]
-; DISABLED_MASKED_STRIDED: pred.load.continue26:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = phi <8 x i8> [ [[TMP76]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP81]], [[PRED_LOAD_IF25]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = extractelement <8 x i1> [[TMP3]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP83]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if27:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE26]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE26]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = phi <8 x i8> [ [[TMP76]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP81]], %[[PRED_LOAD_IF25]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP83]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF27]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = extractelement <8 x i32> [[TMP52]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP84]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = load i8, ptr [[TMP85]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = insertelement <8 x i8> [[TMP82]], i8 [[TMP86]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE28]]
-; DISABLED_MASKED_STRIDED: pred.load.continue28:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = phi <8 x i8> [ [[TMP82]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP87]], [[PRED_LOAD_IF27]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = extractelement <8 x i1> [[TMP3]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP89]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if29:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE28]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE28]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = phi <8 x i8> [ [[TMP82]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP87]], %[[PRED_LOAD_IF27]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP89]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF29]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = extractelement <8 x i32> [[TMP52]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP90]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = load i8, ptr [[TMP91]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = insertelement <8 x i8> [[TMP88]], i8 [[TMP92]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE30]]
-; DISABLED_MASKED_STRIDED: pred.load.continue30:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = phi <8 x i8> [ [[TMP88]], [[PRED_LOAD_CONTINUE28]] ], [ [[TMP93]], [[PRED_LOAD_IF29]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = extractelement <8 x i1> [[TMP3]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP95]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if31:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE30]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE30]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = phi <8 x i8> [ [[TMP88]], %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP93]], %[[PRED_LOAD_IF29]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP95]], label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF31]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = extractelement <8 x i32> [[TMP52]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP96]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = load i8, ptr [[TMP97]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = insertelement <8 x i8> [[TMP94]], i8 [[TMP98]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE32]]
-; DISABLED_MASKED_STRIDED: pred.load.continue32:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = phi <8 x i8> [ [[TMP94]], [[PRED_LOAD_CONTINUE30]] ], [ [[TMP99]], [[PRED_LOAD_IF31]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE32]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE32]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = phi <8 x i8> [ [[TMP94]], %[[PRED_LOAD_CONTINUE30]] ], [ [[TMP99]], %[[PRED_LOAD_IF31]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[TMP51]], <8 x i8> [[TMP100]])
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = extractelement <8 x i1> [[TMP3]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP102]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP103]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP102]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP103]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i8> [[TMP101]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP105]], ptr [[TMP104]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.store.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = extractelement <8 x i1> [[TMP3]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP106]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if33:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP106]], label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF33]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP107]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i8> [[TMP101]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP109]], ptr [[TMP108]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]]
-; DISABLED_MASKED_STRIDED: pred.store.continue34:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i1> [[TMP3]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP110]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if35:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE34]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE34]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP110]], label %[[PRED_STORE_IF35:.*]], label %[[PRED_STORE_CONTINUE36:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF35]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP111]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i8> [[TMP101]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP113]], ptr [[TMP112]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]]
-; DISABLED_MASKED_STRIDED: pred.store.continue36:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i1> [[TMP3]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP114]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if37:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE36]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE36]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP114]], label %[[PRED_STORE_IF37:.*]], label %[[PRED_STORE_CONTINUE38:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF37]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP115]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i8> [[TMP101]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP117]], ptr [[TMP116]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]]
-; DISABLED_MASKED_STRIDED: pred.store.continue38:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i1> [[TMP3]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP118]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if39:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE38]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE38]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP118]], label %[[PRED_STORE_IF39:.*]], label %[[PRED_STORE_CONTINUE40:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF39]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP119]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i8> [[TMP101]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP121]], ptr [[TMP120]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]]
-; DISABLED_MASKED_STRIDED: pred.store.continue40:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i1> [[TMP3]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP122]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if41:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE40]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE40]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP122]], label %[[PRED_STORE_IF41:.*]], label %[[PRED_STORE_CONTINUE42:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF41]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP123]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i8> [[TMP101]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP125]], ptr [[TMP124]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]]
-; DISABLED_MASKED_STRIDED: pred.store.continue42:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i1> [[TMP3]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP126]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if43:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE42]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE42]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP126]], label %[[PRED_STORE_IF43:.*]], label %[[PRED_STORE_CONTINUE44:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF43]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP127]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i8> [[TMP101]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP129]], ptr [[TMP128]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]]
-; DISABLED_MASKED_STRIDED: pred.store.continue44:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i1> [[TMP3]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP130]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if45:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE44]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE44]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP130]], label %[[PRED_STORE_IF45:.*]], label %[[PRED_STORE_CONTINUE46:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF45]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP131]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i8> [[TMP101]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP133]], ptr [[TMP132]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]]
-; DISABLED_MASKED_STRIDED: pred.store.continue46:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE46]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE46]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = sub <8 x i8> zeroinitializer, [[TMP101]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP102]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if47:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP52]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP136]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i8> [[TMP134]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], ptr [[TMP137]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]]
-; DISABLED_MASKED_STRIDED: pred.store.continue48:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP106]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if49:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = extractelement <8 x i32> [[TMP52]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP140]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i8> [[TMP134]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP142]], ptr [[TMP141]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]]
-; DISABLED_MASKED_STRIDED: pred.store.continue50:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP110]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if51:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i32> [[TMP52]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP102]], label %[[PRED_STORE_IF47:.*]], label %[[PRED_STORE_CONTINUE48:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF47]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i32> [[TMP52]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP135]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i8> [[TMP134]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP137]], ptr [[TMP136]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE48]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE48]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP106]], label %[[PRED_STORE_IF49:.*]], label %[[PRED_STORE_CONTINUE50:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF49]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i32> [[TMP52]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = extractelement <8 x i8> [[TMP134]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP140]], ptr [[TMP139]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE50]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE50]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP110]], label %[[PRED_STORE_IF51:.*]], label %[[PRED_STORE_CONTINUE52:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF51]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i32> [[TMP52]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP141]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = extractelement <8 x i8> [[TMP134]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP143]], ptr [[TMP142]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE52]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE52]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP114]], label %[[PRED_STORE_IF53:.*]], label %[[PRED_STORE_CONTINUE54:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF53]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i32> [[TMP52]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP144]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i8> [[TMP134]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i8> [[TMP134]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP146]], ptr [[TMP145]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]]
-; DISABLED_MASKED_STRIDED: pred.store.continue52:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP114]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if53:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i32> [[TMP52]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP148]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i8> [[TMP134]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP150]], ptr [[TMP149]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]]
-; DISABLED_MASKED_STRIDED: pred.store.continue54:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP118]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if55:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i32> [[TMP52]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP152]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i8> [[TMP134]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP154]], ptr [[TMP153]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]]
-; DISABLED_MASKED_STRIDED: pred.store.continue56:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP122]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if57:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i32> [[TMP52]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE54]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE54]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP118]], label %[[PRED_STORE_IF55:.*]], label %[[PRED_STORE_CONTINUE56:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF55]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP52]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP147]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i8> [[TMP134]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP149]], ptr [[TMP148]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE56]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE56]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP122]], label %[[PRED_STORE_IF57:.*]], label %[[PRED_STORE_CONTINUE58:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF57]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP52]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP134]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], ptr [[TMP151]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE58]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE58]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP126]], label %[[PRED_STORE_IF59:.*]], label %[[PRED_STORE_CONTINUE60:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF59]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i32> [[TMP52]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP153]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i8> [[TMP134]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP155]], ptr [[TMP154]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE60]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE60]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP130]], label %[[PRED_STORE_IF61:.*]], label %[[PRED_STORE_CONTINUE62]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF61]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i32> [[TMP52]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP156]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i8> [[TMP134]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i8> [[TMP134]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP158]], ptr [[TMP157]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]]
-; DISABLED_MASKED_STRIDED: pred.store.continue58:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP126]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if59:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i32> [[TMP52]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP160]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i8> [[TMP134]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP162]], ptr [[TMP161]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]]
-; DISABLED_MASKED_STRIDED: pred.store.continue60:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP130]], label [[PRED_STORE_IF61:%.*]], label [[PRED_STORE_CONTINUE62]]
-; DISABLED_MASKED_STRIDED: pred.store.if61:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i32> [[TMP52]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP164]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = extractelement <8 x i8> [[TMP134]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP166]], ptr [[TMP165]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE62]]
-; DISABLED_MASKED_STRIDED: pred.store.continue62:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE62]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE62]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP167:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP167]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP159]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @masked_strided2_unknown_tc(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CMP22:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP22]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; ENABLED_MASKED_STRIDED: vector.ph:
+; ENABLED_MASKED_STRIDED-LABEL: define void @masked_strided2_unknown_tc(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[GUARD:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; ENABLED_MASKED_STRIDED-NEXT: [[CMP22:%.*]] = icmp sgt i32 [[N]], 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP22]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD:%.*]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 [[TMP2]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
-; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP3]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl i32 [[INDEX]], 1
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8> [[STRIDED_VEC3]])
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[TMP2]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = sub <8 x i8> zeroinitializer, [[TMP6]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP6]], <8 x i8> [[TMP7]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP8]], <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8> [[STRIDED_VEC3]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q]], i32 [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = sub <8 x i8> zeroinitializer, [[TMP5]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP5]], <8 x i8> [[TMP7]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -2675,360 +2693,362 @@ for.end:
;}
;
define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %n) optsize {
-; DISABLED_MASKED_STRIDED-LABEL: @unconditional_masked_strided2_unknown_tc(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CMP20:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP20]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; DISABLED_MASKED_STRIDED: vector.ph:
+; DISABLED_MASKED_STRIDED-LABEL: define void @unconditional_masked_strided2_unknown_tc(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; DISABLED_MASKED_STRIDED-NEXT: [[CMP20:%.*]] = icmp sgt i32 [[N]], 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP20]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE60:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE60]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE60:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.load.continue:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = phi <8 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.load.continue2:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE2]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = phi <8 x i8> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.load.continue4:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE4]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = phi <8 x i8> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], %[[PRED_LOAD_IF3]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.load.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = phi <8 x i8> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], %[[PRED_LOAD_IF5]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if7:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF7]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE8]]
-; DISABLED_MASKED_STRIDED: pred.load.continue8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = phi <8 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], %[[PRED_LOAD_IF7]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if9:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF9]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE10]]
-; DISABLED_MASKED_STRIDED: pred.load.continue10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], [[PRED_LOAD_IF9]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = phi <8 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP36]], %[[PRED_LOAD_IF9]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if11:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF11:.*]], label %[[PRED_LOAD_CONTINUE12:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF11]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE12]]
-; DISABLED_MASKED_STRIDED: pred.load.continue12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], [[PRED_LOAD_IF11]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE12]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = phi <8 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE10]] ], [ [[TMP42]], %[[PRED_LOAD_IF11]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if13:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP44]], label %[[PRED_LOAD_IF13:.*]], label %[[PRED_LOAD_CONTINUE14:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF13]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE14]]
-; DISABLED_MASKED_STRIDED: pred.load.continue14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE14]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = or disjoint <8 x i32> [[TMP1]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if15:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF15]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE16]]
-; DISABLED_MASKED_STRIDED: pred.load.continue16:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], [[PRED_LOAD_IF15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE16]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE16]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ poison, %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP55]], %[[PRED_LOAD_IF15]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if17:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP57]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF17]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE18]]
-; DISABLED_MASKED_STRIDED: pred.load.continue18:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], [[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], [[PRED_LOAD_IF17]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE18]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE18]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = phi <8 x i8> [ [[TMP56]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP61]], %[[PRED_LOAD_IF17]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if19:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP63]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF19]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE20]]
-; DISABLED_MASKED_STRIDED: pred.load.continue20:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], [[PRED_LOAD_IF19]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE20]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE20]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = phi <8 x i8> [ [[TMP62]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP67]], %[[PRED_LOAD_IF19]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if21:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP69]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF21]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE22]]
-; DISABLED_MASKED_STRIDED: pred.load.continue22:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], [[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], [[PRED_LOAD_IF21]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE22]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE22]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = phi <8 x i8> [ [[TMP68]], %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP73]], %[[PRED_LOAD_IF21]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if23:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP75]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF23]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE24]]
-; DISABLED_MASKED_STRIDED: pred.load.continue24:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], [[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], [[PRED_LOAD_IF23]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE24]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE24]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = phi <8 x i8> [ [[TMP74]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP79]], %[[PRED_LOAD_IF23]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if25:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP81]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF25]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE26]]
-; DISABLED_MASKED_STRIDED: pred.load.continue26:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], [[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], [[PRED_LOAD_IF25]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE26]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE26]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = phi <8 x i8> [ [[TMP80]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP85]], %[[PRED_LOAD_IF25]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if27:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP87]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF27]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE28]]
-; DISABLED_MASKED_STRIDED: pred.load.continue28:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], [[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], [[PRED_LOAD_IF27]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE28]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE28]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = phi <8 x i8> [ [[TMP86]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP91]], %[[PRED_LOAD_IF27]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
-; DISABLED_MASKED_STRIDED: pred.load.if29:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP93]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_IF29]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE30]]
-; DISABLED_MASKED_STRIDED: pred.load.continue30:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], [[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], [[PRED_LOAD_IF29]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_LOAD_CONTINUE30]]
+; DISABLED_MASKED_STRIDED: [[PRED_LOAD_CONTINUE30]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = phi <8 x i8> [ [[TMP92]], %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP97]], %[[PRED_LOAD_IF29]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[TMP49]], <8 x i8> [[TMP98]])
; DISABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP101]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP103]], ptr [[TMP102]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.store.continue:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if31:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF31]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP107]], ptr [[TMP106]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE32]]
-; DISABLED_MASKED_STRIDED: pred.store.continue32:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE32]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if33:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF33]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], ptr [[TMP110]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]]
-; DISABLED_MASKED_STRIDED: pred.store.continue34:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE34]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE34]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if35:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF35:.*]], label %[[PRED_STORE_CONTINUE36:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF35]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP115]], ptr [[TMP114]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]]
-; DISABLED_MASKED_STRIDED: pred.store.continue36:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE36]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE36]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if37:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF37:.*]], label %[[PRED_STORE_CONTINUE38:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF37]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], ptr [[TMP118]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]]
-; DISABLED_MASKED_STRIDED: pred.store.continue38:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE38]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE38]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if39:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF39:.*]], label %[[PRED_STORE_CONTINUE40:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF39]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP123]], ptr [[TMP122]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]]
-; DISABLED_MASKED_STRIDED: pred.store.continue40:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE40]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE40]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if41:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF41:.*]], label %[[PRED_STORE_CONTINUE42:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF41]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], ptr [[TMP126]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]]
-; DISABLED_MASKED_STRIDED: pred.store.continue42:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE42]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE42]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if43:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF43:.*]], label %[[PRED_STORE_CONTINUE44:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF43]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP131]], ptr [[TMP130]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]]
-; DISABLED_MASKED_STRIDED: pred.store.continue44:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE44]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE44]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = sub <8 x i8> zeroinitializer, [[TMP99]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if45:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP136]], ptr [[TMP135]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]]
-; DISABLED_MASKED_STRIDED: pred.store.continue46:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if47:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP140]], ptr [[TMP139]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]]
-; DISABLED_MASKED_STRIDED: pred.store.continue48:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if49:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label %[[PRED_STORE_IF45:.*]], label %[[PRED_STORE_CONTINUE46:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF45]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP133]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], ptr [[TMP134]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE46]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE46]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP104]], label %[[PRED_STORE_IF47:.*]], label %[[PRED_STORE_CONTINUE48:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF47]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP136]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], ptr [[TMP137]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE48]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE48]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label %[[PRED_STORE_IF49:.*]], label %[[PRED_STORE_CONTINUE50:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF49]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP139]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], ptr [[TMP140]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE50]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE50]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label %[[PRED_STORE_IF51:.*]], label %[[PRED_STORE_CONTINUE52:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF51]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], ptr [[TMP143]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]]
-; DISABLED_MASKED_STRIDED: pred.store.continue50:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if51:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP148]], ptr [[TMP147]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]]
-; DISABLED_MASKED_STRIDED: pred.store.continue52:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if53:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], ptr [[TMP151]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]]
-; DISABLED_MASKED_STRIDED: pred.store.continue54:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if55:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE52]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE52]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label %[[PRED_STORE_IF53:.*]], label %[[PRED_STORE_CONTINUE54:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF53]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP145]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP147]], ptr [[TMP146]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE54]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE54]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label %[[PRED_STORE_IF55:.*]], label %[[PRED_STORE_CONTINUE56:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF55]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP148]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP150]], ptr [[TMP149]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE56]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE56]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label %[[PRED_STORE_IF57:.*]], label %[[PRED_STORE_CONTINUE58:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF57]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP151]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
+; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP153]], ptr [[TMP152]], align 1
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE58]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE58]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label %[[PRED_STORE_IF59:.*]], label %[[PRED_STORE_CONTINUE60]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF59]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], ptr [[TMP155]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]]
-; DISABLED_MASKED_STRIDED: pred.store.continue56:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if57:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], ptr [[TMP159]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]]
-; DISABLED_MASKED_STRIDED: pred.store.continue58:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
-; DISABLED_MASKED_STRIDED: pred.store.if59:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP164]], ptr [[TMP163]], align 1
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]]
-; DISABLED_MASKED_STRIDED: pred.store.continue60:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE60]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE60]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @unconditional_masked_strided2_unknown_tc(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CMP20:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP20]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; ENABLED_MASKED_STRIDED: vector.ph:
+; ENABLED_MASKED_STRIDED-LABEL: define void @unconditional_masked_strided2_unknown_tc(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; ENABLED_MASKED_STRIDED-NEXT: [[CMP20:%.*]] = icmp sgt i32 [[N]], 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP20]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7
; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i32 [[N]], -1
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[INDEX]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IV:%.*]] = or disjoint <8 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8> [[STRIDED_VEC3]])
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = sub <8 x i8> zeroinitializer, [[TMP4]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8> [[STRIDED_VEC3]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = sub <8 x i8> zeroinitializer, [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP6]], label %[[FOR_END]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index c2530eaa2404e..dab0aac63efcf 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -mcpu=skx -S -passes='loop-vectorize,instcombine,simplifycfg,loop-mssa(licm)' -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses -tail-folding-policy=must-fold-tail < %s | FileCheck %s -check-prefix=DISABLED_MASKED_STRIDED
; RUN: opt -mcpu=skx -S -passes='loop-vectorize,instcombine,simplifycfg,loop-mssa(licm)' -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -tail-folding-policy=must-fold-tail < %s | FileCheck %s -check-prefix=ENABLED_MASKED_STRIDED
@@ -18,74 +18,76 @@ target triple = "x86_64-unknown-linux-gnu"
; (relates to the testcase in PR50566)
define void @test1(ptr noalias nocapture %points, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y) {
-; DISABLED_MASKED_STRIDED-LABEL: @test1(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i64 [[INDEX]]
+; DISABLED_MASKED_STRIDED-LABEL: define void @test1(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias captures(none) [[POINTS:%.*]], ptr noalias readonly captures(none) [[X:%.*]], ptr noalias readonly captures(none) [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 2)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS:%.*]], i64 [[TMP2]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP6]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP10]], ptr [[TMP3]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP11]], ptr [[TMP5]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP12]], ptr [[TMP7]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP13]], ptr [[TMP9]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i64 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 2)
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP5]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP5]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP5]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP5]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP6]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP7]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP1]], ptr [[TMP10]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP2]], ptr [[TMP11]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP3]], ptr [[TMP12]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP4]], ptr [[TMP13]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i64 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP14]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = or disjoint <4 x i64> [[TMP1]], splat (i64 1)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP15]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP15]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP15]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP15]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP18]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP20]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP22]]
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP24]], ptr [[TMP17]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP25]], ptr [[TMP19]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP26]], ptr [[TMP21]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP27]], ptr [[TMP23]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = or disjoint <4 x i64> [[TMP5]], splat (i64 1)
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP19]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP19]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP19]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[TMP19]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP20]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP22]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP23]]
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP15]], ptr [[TMP24]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP16]], ptr [[TMP25]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP17]], ptr [[TMP26]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP18]], ptr [[TMP27]], align 2
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP28]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @test1(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-LABEL: define void @test1(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias captures(none) [[POINTS:%.*]], ptr noalias readonly captures(none) [[X:%.*]], ptr noalias readonly captures(none) [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 3
-; ENABLED_MASKED_STRIDED-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[POINTS:%.*]], i64 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[DOTIDX:%.*]] = shl nsw i64 [[INDEX]], 3
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[POINTS]], i64 [[DOTIDX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD1]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[GEP]], <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>)
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP1]], <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>)
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -121,139 +123,141 @@ for.end:
; }
define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y) {
-; DISABLED_MASKED_STRIDED-LABEL: @test2(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS:%.*]], 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP15]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; DISABLED_MASKED_STRIDED: vector.ph:
+; DISABLED_MASKED_STRIDED-LABEL: define void @test2(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias captures(none) [[POINTS:%.*]], i32 [[NUMPOINTS:%.*]], ptr noalias readonly captures(none) [[X:%.*]], ptr noalias readonly captures(none) [[Y:%.*]]) #[[ATTR0]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; DISABLED_MASKED_STRIDED-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS]], 0
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP15]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[WIDE_TRIP_COUNT]], 3
; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 4294967292
; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE15:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE15:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE15]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i64 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP1]], <4 x i1> [[TMP0]], <4 x i16> poison)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl nsw <4 x i64> [[VEC_IND]], splat (i64 2)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS:%.*]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP6]], ptr [[TMP5]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.store.continue:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP10]], ptr [[TMP9]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.store.continue2:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE2]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP14]], ptr [[TMP13]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.store.continue4:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE4]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP18]], ptr [[TMP17]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.store.continue6:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i64 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE6]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i64 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP19]], <4 x i1> [[TMP0]], <4 x i16> poison)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = or disjoint <4 x i64> [[TMP2]], splat (i64 1)
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if8:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP22]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP24]], ptr [[TMP23]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE9]]
-; DISABLED_MASKED_STRIDED: pred.store.continue9:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if10:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP26]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP28]], ptr [[TMP27]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE11]]
-; DISABLED_MASKED_STRIDED: pred.store.continue11:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if12:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF8]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP23]], ptr [[TMP22]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE9]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE9]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF10]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP24]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP26]], ptr [[TMP25]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE11]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE11]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF12]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP29]], ptr [[TMP28]], align 2
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE13]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE13]]:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF14]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP30]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP32]], ptr [[TMP31]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE13]]
-; DISABLED_MASKED_STRIDED: pred.store.continue13:
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15]]
-; DISABLED_MASKED_STRIDED: pred.store.if14:
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP34]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP36]], ptr [[TMP35]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE15]]
-; DISABLED_MASKED_STRIDED: pred.store.continue15:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE15]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE15]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP37]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end.loopexit:
-; DISABLED_MASKED_STRIDED-NEXT: br label [[FOR_END]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP33]], label %[[FOR_END_LOOPEXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @test2(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS:%.*]], 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP15]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
-; ENABLED_MASKED_STRIDED: vector.ph:
+; ENABLED_MASKED_STRIDED-LABEL: define void @test2(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias captures(none) [[POINTS:%.*]], i32 [[NUMPOINTS:%.*]], ptr noalias readonly captures(none) [[X:%.*]], ptr noalias readonly captures(none) [[Y:%.*]]) #[[ATTR0]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*:]]
+; ENABLED_MASKED_STRIDED-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS]], 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP15]], label %[[VECTOR_PH:.*]], label %[[FOR_END:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[WIDE_TRIP_COUNT]], 3
; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 4294967292
; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP1]], <4 x i1> [[TMP0]], <4 x i16> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl nsw i64 [[INDEX]], 3
-; ENABLED_MASKED_STRIDED-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[POINTS:%.*]], i64 [[TMP2]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[DOTIDX:%.*]] = shl nsw i64 [[INDEX]], 3
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[POINTS]], i64 [[DOTIDX]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[TMP0]], <4 x i16> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_MASKED_LOAD]], <4 x i16> [[WIDE_MASKED_LOAD3]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP0]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[GEP]], <16 x i1> [[TMP5]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP2]], <16 x i1> [[TMP4]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP6]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end.loopexit:
-; ENABLED_MASKED_STRIDED-NEXT: br label [[FOR_END]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label %[[FOR_END_LOOPEXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
@@ -298,110 +302,112 @@ for.end:
; points[i*3] = x[i];
; }
define void @test(ptr noalias nocapture %points, ptr noalias nocapture readonly %x, ptr noalias nocapture readnone %y) {
-; DISABLED_MASKED_STRIDED-LABEL: @test(
-; DISABLED_MASKED_STRIDED-NEXT: entry:
-; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; DISABLED_MASKED_STRIDED: vector.body:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i64 [[INDEX]]
+; DISABLED_MASKED_STRIDED-LABEL: define void @test(
+; DISABLED_MASKED_STRIDED-SAME: ptr noalias captures(none) [[POINTS:%.*]], ptr noalias readonly captures(none) [[X:%.*]], ptr noalias readnone captures(none) [[Y:%.*]]) #[[ATTR0]] {
+; DISABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], splat (i64 3)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS:%.*]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP6]], ptr [[TMP5]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; DISABLED_MASKED_STRIDED: pred.store.continue:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if1:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP10]], ptr [[TMP9]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; DISABLED_MASKED_STRIDED: pred.store.continue2:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE2]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
-; DISABLED_MASKED_STRIDED: pred.store.if3:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP14]], ptr [[TMP13]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; DISABLED_MASKED_STRIDED: pred.store.continue4:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE4]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.store.if5:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP18]], ptr [[TMP17]], align 2
-; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; DISABLED_MASKED_STRIDED: pred.store.continue6:
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE6]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP19]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; DISABLED_MASKED_STRIDED: for.end:
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP19]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DISABLED_MASKED_STRIDED: [[FOR_END]]:
; DISABLED_MASKED_STRIDED-NEXT: ret void
;
-; ENABLED_MASKED_STRIDED-LABEL: @test(
-; ENABLED_MASKED_STRIDED-NEXT: entry:
-; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
-; ENABLED_MASKED_STRIDED: vector.body:
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-LABEL: define void @test(
+; ENABLED_MASKED_STRIDED-SAME: ptr noalias captures(none) [[POINTS:%.*]], ptr noalias readonly captures(none) [[X:%.*]], ptr noalias readnone captures(none) [[Y:%.*]]) #[[ATTR0]] {
+; ENABLED_MASKED_STRIDED-NEXT: [[ENTRY:.*]]:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
+; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], splat (i64 3)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS:%.*]], i64 [[TMP4]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: store i16 [[TMP6]], ptr [[TMP5]], align 2
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]]
-; ENABLED_MASKED_STRIDED: pred.store.continue:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if1:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF1]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
; ENABLED_MASKED_STRIDED-NEXT: store i16 [[TMP10]], ptr [[TMP9]], align 2
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; ENABLED_MASKED_STRIDED: pred.store.continue2:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE2]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
-; ENABLED_MASKED_STRIDED: pred.store.if3:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF3]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
; ENABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
; ENABLED_MASKED_STRIDED-NEXT: store i16 [[TMP14]], ptr [[TMP13]], align 2
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; ENABLED_MASKED_STRIDED: pred.store.continue4:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE4]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
-; ENABLED_MASKED_STRIDED: pred.store.if5:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_IF5]]:
; ENABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
; ENABLED_MASKED_STRIDED-NEXT: store i16 [[TMP18]], ptr [[TMP17]], align 2
-; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; ENABLED_MASKED_STRIDED: pred.store.continue6:
+; ENABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; ENABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE6]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP19]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; ENABLED_MASKED_STRIDED: for.end:
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP19]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; ENABLED_MASKED_STRIDED: [[FOR_END]]:
; ENABLED_MASKED_STRIDED-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
index 2a55784f633e3..d0ab61280cd09 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -mattr=avx -force-vector-width=2 -force-vector-interleave=1 -passes=loop-vectorize,simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
; RUN: opt -mcpu=skylake-avx512 -S -force-vector-width=8 -force-vector-interleave=1 -passes=loop-vectorize < %s | FileCheck %s --check-prefix=SINK-GATHER
@@ -10,133 +10,135 @@ target triple = "x86_64-apple-macosx10.8.0"
; operand to become non-uniform.
;
define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
-; CHECK-LABEL: @predicated_sdiv_masked_load(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0
+; CHECK-LABEL: define i32 @predicated_sdiv_masked_load(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[X:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SDIV_CONTINUE2:%.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP17:%.*]], [[PRED_SDIV_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 4 [[TMP3]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> poison)
-; CHECK-NEXT: br i1 [[C]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
-; CHECK: pred.sdiv.if:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X:%.*]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE]]
-; CHECK: pred.sdiv.continue:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_SDIV_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_SDIV_IF1:%.*]], label [[PRED_SDIV_CONTINUE2]]
-; CHECK: pred.sdiv.if1:
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = sdiv i32 [[TMP11]], [[X]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP12]], i64 1
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE2]]
-; CHECK: pred.sdiv.continue2:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i32> [ [[TMP9]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_SDIV_IF1]] ]
-; CHECK-NEXT: [[TMP15:%.*]] = add nsw <2 x i32> [[TMP14]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP15]], <2 x i32> [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP17]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP11:%.*]], %[[PRED_SDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 4 [[TMP1]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> poison)
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK: [[PRED_SDIV_IF]]:
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[TMP2]], [[X]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK: [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2]]
+; CHECK: [[PRED_SDIV_IF1]]:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP7]], i64 1
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
+; CHECK: [[PRED_SDIV_CONTINUE2]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <2 x i32> [[TMP9]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP10]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP17]])
-; CHECK-NEXT: ret i32 [[TMP19]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP11]])
+; CHECK-NEXT: ret i32 [[TMP13]]
;
-; SINK-GATHER-LABEL: @predicated_sdiv_masked_load(
-; SINK-GATHER-NEXT: entry:
-; SINK-GATHER-NEXT: br label [[VECTOR_PH:%.*]]
-; SINK-GATHER: vector.ph:
-; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C:%.*]], i64 0
+; SINK-GATHER-LABEL: define i32 @predicated_sdiv_masked_load(
+; SINK-GATHER-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[X:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; SINK-GATHER-NEXT: [[ENTRY:.*:]]
+; SINK-GATHER-NEXT: br label %[[VECTOR_PH:.*]]
+; SINK-GATHER: [[VECTOR_PH]]:
+; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
; SINK-GATHER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; SINK-GATHER-NEXT: br label [[VECTOR_BODY:%.*]]
-; SINK-GATHER: vector.body:
-; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SDIV_CONTINUE14:%.*]] ]
-; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_SDIV_CONTINUE14]] ]
-; SINK-GATHER-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; SINK-GATHER-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
-; SINK-GATHER-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[INDEX]]
-; SINK-GATHER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP3]], <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
-; SINK-GATHER: pred.sdiv.if:
-; SINK-GATHER-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 0
-; SINK-GATHER-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X:%.*]]
-; SINK-GATHER-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> poison, i32 [[TMP7]], i64 0
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE]]
-; SINK-GATHER: pred.sdiv.continue:
-; SINK-GATHER-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_SDIV_IF]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF1:%.*]], label [[PRED_SDIV_CONTINUE2:%.*]]
-; SINK-GATHER: pred.sdiv.if1:
-; SINK-GATHER-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 1
-; SINK-GATHER-NEXT: [[TMP12:%.*]] = sdiv i32 [[TMP11]], [[X]]
-; SINK-GATHER-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[TMP12]], i64 1
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE2]]
-; SINK-GATHER: pred.sdiv.continue2:
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = phi <8 x i32> [ [[TMP9]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_SDIV_IF1]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4:%.*]]
-; SINK-GATHER: pred.sdiv.if3:
-; SINK-GATHER-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 2
-; SINK-GATHER-NEXT: [[TMP17:%.*]] = sdiv i32 [[TMP16]], [[X]]
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[TMP17]], i64 2
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE4]]
-; SINK-GATHER: pred.sdiv.continue4:
-; SINK-GATHER-NEXT: [[TMP19:%.*]] = phi <8 x i32> [ [[TMP14]], [[PRED_SDIV_CONTINUE2]] ], [ [[TMP18]], [[PRED_SDIV_IF3]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF5:%.*]], label [[PRED_SDIV_CONTINUE6:%.*]]
-; SINK-GATHER: pred.sdiv.if5:
-; SINK-GATHER-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 3
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = sdiv i32 [[TMP21]], [[X]]
-; SINK-GATHER-NEXT: [[TMP23:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP22]], i64 3
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE6]]
-; SINK-GATHER: pred.sdiv.continue6:
-; SINK-GATHER-NEXT: [[TMP24:%.*]] = phi <8 x i32> [ [[TMP19]], [[PRED_SDIV_CONTINUE4]] ], [ [[TMP23]], [[PRED_SDIV_IF5]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF7:%.*]], label [[PRED_SDIV_CONTINUE8:%.*]]
-; SINK-GATHER: pred.sdiv.if7:
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 4
+; SINK-GATHER-NEXT: br label %[[VECTOR_BODY:.*]]
+; SINK-GATHER: [[VECTOR_BODY]]:
+; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE14:.*]] ]
+; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[PRED_SDIV_CONTINUE14]] ]
+; SINK-GATHER-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; SINK-GATHER-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
+; SINK-GATHER-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; SINK-GATHER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP1]], <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF]]:
+; SINK-GATHER-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 0
+; SINK-GATHER-NEXT: [[TMP3:%.*]] = sdiv i32 [[TMP2]], [[X]]
+; SINK-GATHER-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE]]:
+; SINK-GATHER-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF1]]:
+; SINK-GATHER-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 1
+; SINK-GATHER-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X]]
+; SINK-GATHER-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[TMP7]], i64 1
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE2]]:
+; SINK-GATHER-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF3]]:
+; SINK-GATHER-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 2
+; SINK-GATHER-NEXT: [[TMP11:%.*]] = sdiv i32 [[TMP10]], [[X]]
+; SINK-GATHER-NEXT: [[TMP12:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[TMP11]], i64 2
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE4]]:
+; SINK-GATHER-NEXT: [[TMP13:%.*]] = phi <8 x i32> [ [[TMP9]], %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP12]], %[[PRED_SDIV_IF3]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF5]]:
+; SINK-GATHER-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 3
+; SINK-GATHER-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP14]], [[X]]
+; SINK-GATHER-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP13]], i32 [[TMP15]], i64 3
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE6]]:
+; SINK-GATHER-NEXT: [[TMP17:%.*]] = phi <8 x i32> [ [[TMP13]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP16]], %[[PRED_SDIV_IF5]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF7:.*]], label %[[PRED_SDIV_CONTINUE8:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF7]]:
+; SINK-GATHER-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 4
+; SINK-GATHER-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP18]], [[X]]
+; SINK-GATHER-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[TMP19]], i64 4
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE8]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE8]]:
+; SINK-GATHER-NEXT: [[TMP21:%.*]] = phi <8 x i32> [ [[TMP17]], %[[PRED_SDIV_CONTINUE6]] ], [ [[TMP20]], %[[PRED_SDIV_IF7]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF9:.*]], label %[[PRED_SDIV_CONTINUE10:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF9]]:
+; SINK-GATHER-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 5
+; SINK-GATHER-NEXT: [[TMP23:%.*]] = sdiv i32 [[TMP22]], [[X]]
+; SINK-GATHER-NEXT: [[TMP24:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP23]], i64 5
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE10]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE10]]:
+; SINK-GATHER-NEXT: [[TMP25:%.*]] = phi <8 x i32> [ [[TMP21]], %[[PRED_SDIV_CONTINUE8]] ], [ [[TMP24]], %[[PRED_SDIV_IF9]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF11:.*]], label %[[PRED_SDIV_CONTINUE12:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF11]]:
+; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 6
; SINK-GATHER-NEXT: [[TMP27:%.*]] = sdiv i32 [[TMP26]], [[X]]
-; SINK-GATHER-NEXT: [[TMP28:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP27]], i64 4
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE8]]
-; SINK-GATHER: pred.sdiv.continue8:
-; SINK-GATHER-NEXT: [[TMP29:%.*]] = phi <8 x i32> [ [[TMP24]], [[PRED_SDIV_CONTINUE6]] ], [ [[TMP28]], [[PRED_SDIV_IF7]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF9:%.*]], label [[PRED_SDIV_CONTINUE10:%.*]]
-; SINK-GATHER: pred.sdiv.if9:
-; SINK-GATHER-NEXT: [[TMP31:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 5
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = sdiv i32 [[TMP31]], [[X]]
-; SINK-GATHER-NEXT: [[TMP33:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP32]], i64 5
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE10]]
-; SINK-GATHER: pred.sdiv.continue10:
-; SINK-GATHER-NEXT: [[TMP34:%.*]] = phi <8 x i32> [ [[TMP29]], [[PRED_SDIV_CONTINUE8]] ], [ [[TMP33]], [[PRED_SDIV_IF9]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF11:%.*]], label [[PRED_SDIV_CONTINUE12:%.*]]
-; SINK-GATHER: pred.sdiv.if11:
-; SINK-GATHER-NEXT: [[TMP36:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 6
-; SINK-GATHER-NEXT: [[TMP37:%.*]] = sdiv i32 [[TMP36]], [[X]]
-; SINK-GATHER-NEXT: [[TMP38:%.*]] = insertelement <8 x i32> [[TMP34]], i32 [[TMP37]], i64 6
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE12]]
-; SINK-GATHER: pred.sdiv.continue12:
-; SINK-GATHER-NEXT: [[TMP39:%.*]] = phi <8 x i32> [ [[TMP34]], [[PRED_SDIV_CONTINUE10]] ], [ [[TMP38]], [[PRED_SDIV_IF11]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label [[PRED_SDIV_IF13:%.*]], label [[PRED_SDIV_CONTINUE14]]
-; SINK-GATHER: pred.sdiv.if13:
-; SINK-GATHER-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 7
-; SINK-GATHER-NEXT: [[TMP42:%.*]] = sdiv i32 [[TMP41]], [[X]]
-; SINK-GATHER-NEXT: [[TMP43:%.*]] = insertelement <8 x i32> [[TMP39]], i32 [[TMP42]], i64 7
-; SINK-GATHER-NEXT: br label [[PRED_SDIV_CONTINUE14]]
-; SINK-GATHER: pred.sdiv.continue14:
-; SINK-GATHER-NEXT: [[TMP44:%.*]] = phi <8 x i32> [ [[TMP39]], [[PRED_SDIV_CONTINUE12]] ], [ [[TMP43]], [[PRED_SDIV_IF13]] ]
-; SINK-GATHER-NEXT: [[TMP45:%.*]] = add nsw <8 x i32> [[TMP44]], [[WIDE_LOAD]]
-; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP45]], <8 x i32> [[WIDE_LOAD]]
-; SINK-GATHER-NEXT: [[TMP47]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
+; SINK-GATHER-NEXT: [[TMP28:%.*]] = insertelement <8 x i32> [[TMP25]], i32 [[TMP27]], i64 6
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE12]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE12]]:
+; SINK-GATHER-NEXT: [[TMP29:%.*]] = phi <8 x i32> [ [[TMP25]], %[[PRED_SDIV_CONTINUE10]] ], [ [[TMP28]], %[[PRED_SDIV_IF11]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF13:.*]], label %[[PRED_SDIV_CONTINUE14]]
+; SINK-GATHER: [[PRED_SDIV_IF13]]:
+; SINK-GATHER-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 7
+; SINK-GATHER-NEXT: [[TMP31:%.*]] = sdiv i32 [[TMP30]], [[X]]
+; SINK-GATHER-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP31]], i64 7
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE14]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE14]]:
+; SINK-GATHER-NEXT: [[TMP33:%.*]] = phi <8 x i32> [ [[TMP29]], %[[PRED_SDIV_CONTINUE12]] ], [ [[TMP32]], %[[PRED_SDIV_IF13]] ]
+; SINK-GATHER-NEXT: [[TMP34:%.*]] = add nsw <8 x i32> [[TMP33]], [[WIDE_LOAD]]
+; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP34]], <8 x i32> [[WIDE_LOAD]]
+; SINK-GATHER-NEXT: [[TMP35]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
; SINK-GATHER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; SINK-GATHER-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; SINK-GATHER-NEXT: br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; SINK-GATHER: middle.block:
-; SINK-GATHER-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP47]])
-; SINK-GATHER-NEXT: br label [[FOR_INC:%.*]]
-; SINK-GATHER: for.end:
-; SINK-GATHER-NEXT: ret i32 [[TMP49]]
+; SINK-GATHER-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; SINK-GATHER-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SINK-GATHER: [[MIDDLE_BLOCK]]:
+; SINK-GATHER-NEXT: [[TMP37:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP35]])
+; SINK-GATHER-NEXT: br label %[[FOR_END:.*]]
+; SINK-GATHER: [[FOR_END]]:
+; SINK-GATHER-NEXT: ret i32 [[TMP37]]
;
entry:
br label %for.body
@@ -171,203 +173,205 @@ for.end:
; instead scalarized if Cost-Model so decided as part of its
; sink-scalar-operands optimization for predicated instructions.
define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
-; CHECK-LABEL: @scalarize_and_sink_gather(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; CHECK-LABEL: define i32 @scalarize_and_sink_gather(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[C:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i32> poison, i32 [[X:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT1]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE4:%.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_UDIV_CONTINUE4]] ]
-; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[PRED_UDIV_CONTINUE2]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
+; CHECK: [[PRED_UDIV_IF]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 777
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP4]], [[X]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
-; CHECK: pred.udiv.if1:
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 777
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[X]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
+; CHECK: [[PRED_UDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
+; CHECK: [[PRED_UDIV_IF1]]:
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP7]], 777
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = udiv i32 [[TMP12]], [[X]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP13]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; CHECK: pred.udiv.continue2:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP14]], [[PRED_UDIV_IF3]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], <2 x i32> [[TMP16]], <2 x i32> [[BROADCAST_SPLAT4]]
-; CHECK-NEXT: [[TMP18]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 777
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[X]]
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
+; CHECK: [[PRED_UDIV_CONTINUE2]]:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP13]], <2 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP14]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP18]])
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP14]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T6:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[T6:%.*]], %[[FOR_INC]] ]
; CHECK-NEXT: [[I7:%.*]] = mul i64 [[I]], 777
-; CHECK-NEXT: br i1 [[TMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; CHECK: if.then:
+; CHECK-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; CHECK: [[IF_THEN]]:
; CHECK-NEXT: [[T0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I7]]
; CHECK-NEXT: [[T2:%.*]] = load i32, ptr [[T0]], align 4
; CHECK-NEXT: [[T4:%.*]] = udiv i32 [[T2]], [[X]]
-; CHECK-NEXT: br label [[FOR_INC]]
-; CHECK: for.inc:
-; CHECK-NEXT: [[T5:%.*]] = phi i32 [ [[X]], [[FOR_BODY]] ], [ [[T4]], [[IF_THEN]] ]
+; CHECK-NEXT: br label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[T5:%.*]] = phi i32 [ [[X]], %[[FOR_BODY]] ], [ [[T4]], %[[IF_THEN]] ]
; CHECK-NEXT: [[T6]] = add i32 [[R]], [[T5]]
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: for.end:
-; CHECK-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], [[FOR_INC]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[T7]]
;
-; SINK-GATHER-LABEL: @scalarize_and_sink_gather(
-; SINK-GATHER-NEXT: entry:
-; SINK-GATHER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; SINK-GATHER-LABEL: define i32 @scalarize_and_sink_gather(
+; SINK-GATHER-SAME: ptr [[A:%.*]], i1 [[C:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; SINK-GATHER-NEXT: [[ENTRY:.*]]:
+; SINK-GATHER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; SINK-GATHER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8
-; SINK-GATHER-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; SINK-GATHER: vector.ph:
+; SINK-GATHER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SINK-GATHER: [[VECTOR_PH]]:
; SINK-GATHER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 8
; SINK-GATHER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[X:%.*]], i64 0
-; SINK-GATHER-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; SINK-GATHER-NEXT: br label [[VECTOR_BODY:%.*]]
-; SINK-GATHER: vector.body:
-; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE16:%.*]] ]
-; SINK-GATHER-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE16]] ]
-; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP66:%.*]], [[PRED_UDIV_CONTINUE16]] ]
+; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; SINK-GATHER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; SINK-GATHER-NEXT: br label %[[VECTOR_BODY:.*]]
+; SINK-GATHER: [[VECTOR_BODY]]:
+; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE14:.*]] ]
+; SINK-GATHER-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_UDIV_CONTINUE14]] ]
+; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP49:%.*]], %[[PRED_UDIV_CONTINUE14]] ]
; SINK-GATHER-NEXT: [[TMP0:%.*]] = mul <8 x i64> [[VEC_IND]], splat (i64 777)
-; SINK-GATHER-NEXT: br i1 [[TMP1:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; SINK-GATHER: pred.udiv.if:
-; SINK-GATHER-NEXT: [[TMP2:%.*]] = extractelement <8 x i64> [[TMP0]], i64 0
-; SINK-GATHER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP2]]
-; SINK-GATHER-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
-; SINK-GATHER-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP4]], [[X]]
-; SINK-GATHER-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> poison, i32 [[TMP5]], i64 0
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; SINK-GATHER: pred.udiv.continue:
-; SINK-GATHER-NEXT: [[TMP8:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
-; SINK-GATHER: pred.udiv.if1:
-; SINK-GATHER-NEXT: [[TMP10:%.*]] = extractelement <8 x i64> [[TMP0]], i64 1
-; SINK-GATHER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
-; SINK-GATHER-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
-; SINK-GATHER-NEXT: [[TMP13:%.*]] = udiv i32 [[TMP12]], [[X]]
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[TMP13]], i64 1
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; SINK-GATHER: pred.udiv.continue2:
-; SINK-GATHER-NEXT: [[TMP16:%.*]] = phi <8 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP14]], [[PRED_UDIV_IF5]] ]
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
-; SINK-GATHER: pred.udiv.if3:
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = extractelement <8 x i64> [[TMP0]], i64 2
-; SINK-GATHER-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP18]]
-; SINK-GATHER-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4
-; SINK-GATHER-NEXT: [[TMP21:%.*]] = udiv i32 [[TMP20]], [[X]]
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = insertelement <8 x i32> [[TMP16]], i32 [[TMP21]], i64 2
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE6]]
-; SINK-GATHER: pred.udiv.continue4:
-; SINK-GATHER-NEXT: [[TMP24:%.*]] = phi <8 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP22]], [[PRED_UDIV_IF6]] ]
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
-; SINK-GATHER: pred.udiv.if5:
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i64> [[TMP0]], i64 3
-; SINK-GATHER-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP26]]
-; SINK-GATHER-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP27]], align 4
-; SINK-GATHER-NEXT: [[TMP29:%.*]] = udiv i32 [[TMP28]], [[X]]
-; SINK-GATHER-NEXT: [[TMP30:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP29]], i64 3
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE8]]
-; SINK-GATHER: pred.udiv.continue6:
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = phi <8 x i32> [ [[TMP24]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP30]], [[PRED_UDIV_IF7]] ]
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
-; SINK-GATHER: pred.udiv.if7:
-; SINK-GATHER-NEXT: [[TMP34:%.*]] = extractelement <8 x i64> [[TMP0]], i64 4
-; SINK-GATHER-NEXT: [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP34]]
-; SINK-GATHER-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP35]], align 4
-; SINK-GATHER-NEXT: [[TMP37:%.*]] = udiv i32 [[TMP36]], [[X]]
-; SINK-GATHER-NEXT: [[TMP38:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP37]], i64 4
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE10]]
-; SINK-GATHER: pred.udiv.continue8:
-; SINK-GATHER-NEXT: [[TMP40:%.*]] = phi <8 x i32> [ [[TMP32]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP38]], [[PRED_UDIV_IF9]] ]
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
-; SINK-GATHER: pred.udiv.if9:
-; SINK-GATHER-NEXT: [[TMP42:%.*]] = extractelement <8 x i64> [[TMP0]], i64 5
-; SINK-GATHER-NEXT: [[TMP43:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP42]]
-; SINK-GATHER-NEXT: [[TMP44:%.*]] = load i32, ptr [[TMP43]], align 4
-; SINK-GATHER-NEXT: [[TMP45:%.*]] = udiv i32 [[TMP44]], [[X]]
-; SINK-GATHER-NEXT: [[TMP46:%.*]] = insertelement <8 x i32> [[TMP40]], i32 [[TMP45]], i64 5
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE12]]
-; SINK-GATHER: pred.udiv.continue10:
-; SINK-GATHER-NEXT: [[TMP48:%.*]] = phi <8 x i32> [ [[TMP40]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP46]], [[PRED_UDIV_IF11]] ]
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
-; SINK-GATHER: pred.udiv.if11:
-; SINK-GATHER-NEXT: [[TMP50:%.*]] = extractelement <8 x i64> [[TMP0]], i64 6
-; SINK-GATHER-NEXT: [[TMP51:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP50]]
-; SINK-GATHER-NEXT: [[TMP52:%.*]] = load i32, ptr [[TMP51]], align 4
-; SINK-GATHER-NEXT: [[TMP53:%.*]] = udiv i32 [[TMP52]], [[X]]
-; SINK-GATHER-NEXT: [[TMP54:%.*]] = insertelement <8 x i32> [[TMP48]], i32 [[TMP53]], i64 6
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE14]]
-; SINK-GATHER: pred.udiv.continue12:
-; SINK-GATHER-NEXT: [[TMP56:%.*]] = phi <8 x i32> [ [[TMP48]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP54]], [[PRED_UDIV_IF13]] ]
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16]]
-; SINK-GATHER: pred.udiv.if13:
-; SINK-GATHER-NEXT: [[TMP58:%.*]] = extractelement <8 x i64> [[TMP0]], i64 7
-; SINK-GATHER-NEXT: [[TMP59:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP58]]
-; SINK-GATHER-NEXT: [[TMP60:%.*]] = load i32, ptr [[TMP59]], align 4
-; SINK-GATHER-NEXT: [[TMP61:%.*]] = udiv i32 [[TMP60]], [[X]]
-; SINK-GATHER-NEXT: [[TMP62:%.*]] = insertelement <8 x i32> [[TMP56]], i32 [[TMP61]], i64 7
-; SINK-GATHER-NEXT: br label [[PRED_UDIV_CONTINUE16]]
-; SINK-GATHER: pred.udiv.continue14:
-; SINK-GATHER-NEXT: [[TMP64:%.*]] = phi <8 x i32> [ [[TMP56]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP62]], [[PRED_UDIV_IF15]] ]
-; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], <8 x i32> [[TMP64]], <8 x i32> [[BROADCAST_SPLAT16]]
-; SINK-GATHER-NEXT: [[TMP66]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
+; SINK-GATHER: [[PRED_UDIV_IF]]:
+; SINK-GATHER-NEXT: [[TMP1:%.*]] = extractelement <8 x i64> [[TMP0]], i64 0
+; SINK-GATHER-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; SINK-GATHER-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
+; SINK-GATHER-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[X]]
+; SINK-GATHER-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i64 0
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE]]:
+; SINK-GATHER-NEXT: [[TMP6:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]
+; SINK-GATHER: [[PRED_UDIV_IF1]]:
+; SINK-GATHER-NEXT: [[TMP7:%.*]] = extractelement <8 x i64> [[TMP0]], i64 1
+; SINK-GATHER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
+; SINK-GATHER-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+; SINK-GATHER-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP9]], [[X]]
+; SINK-GATHER-NEXT: [[TMP11:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP10]], i64 1
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE2]]:
+; SINK-GATHER-NEXT: [[TMP12:%.*]] = phi <8 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]
+; SINK-GATHER: [[PRED_UDIV_IF3]]:
+; SINK-GATHER-NEXT: [[TMP13:%.*]] = extractelement <8 x i64> [[TMP0]], i64 2
+; SINK-GATHER-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]]
+; SINK-GATHER-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
+; SINK-GATHER-NEXT: [[TMP16:%.*]] = udiv i32 [[TMP15]], [[X]]
+; SINK-GATHER-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[TMP16]], i64 2
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE4]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE4]]:
+; SINK-GATHER-NEXT: [[TMP18:%.*]] = phi <8 x i32> [ [[TMP12]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP17]], %[[PRED_UDIV_IF3]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6:.*]]
+; SINK-GATHER: [[PRED_UDIV_IF5]]:
+; SINK-GATHER-NEXT: [[TMP19:%.*]] = extractelement <8 x i64> [[TMP0]], i64 3
+; SINK-GATHER-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
+; SINK-GATHER-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
+; SINK-GATHER-NEXT: [[TMP22:%.*]] = udiv i32 [[TMP21]], [[X]]
+; SINK-GATHER-NEXT: [[TMP23:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP22]], i64 3
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE6]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE6]]:
+; SINK-GATHER-NEXT: [[TMP24:%.*]] = phi <8 x i32> [ [[TMP18]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP23]], %[[PRED_UDIV_IF5]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF7:.*]], label %[[PRED_UDIV_CONTINUE8:.*]]
+; SINK-GATHER: [[PRED_UDIV_IF7]]:
+; SINK-GATHER-NEXT: [[TMP25:%.*]] = extractelement <8 x i64> [[TMP0]], i64 4
+; SINK-GATHER-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP25]]
+; SINK-GATHER-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4
+; SINK-GATHER-NEXT: [[TMP28:%.*]] = udiv i32 [[TMP27]], [[X]]
+; SINK-GATHER-NEXT: [[TMP29:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP28]], i64 4
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE8]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE8]]:
+; SINK-GATHER-NEXT: [[TMP30:%.*]] = phi <8 x i32> [ [[TMP24]], %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP29]], %[[PRED_UDIV_IF7]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF9:.*]], label %[[PRED_UDIV_CONTINUE10:.*]]
+; SINK-GATHER: [[PRED_UDIV_IF9]]:
+; SINK-GATHER-NEXT: [[TMP31:%.*]] = extractelement <8 x i64> [[TMP0]], i64 5
+; SINK-GATHER-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP31]]
+; SINK-GATHER-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP32]], align 4
+; SINK-GATHER-NEXT: [[TMP34:%.*]] = udiv i32 [[TMP33]], [[X]]
+; SINK-GATHER-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP30]], i32 [[TMP34]], i64 5
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE10]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE10]]:
+; SINK-GATHER-NEXT: [[TMP36:%.*]] = phi <8 x i32> [ [[TMP30]], %[[PRED_UDIV_CONTINUE8]] ], [ [[TMP35]], %[[PRED_UDIV_IF9]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF11:.*]], label %[[PRED_UDIV_CONTINUE12:.*]]
+; SINK-GATHER: [[PRED_UDIV_IF11]]:
+; SINK-GATHER-NEXT: [[TMP37:%.*]] = extractelement <8 x i64> [[TMP0]], i64 6
+; SINK-GATHER-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP37]]
+; SINK-GATHER-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4
+; SINK-GATHER-NEXT: [[TMP40:%.*]] = udiv i32 [[TMP39]], [[X]]
+; SINK-GATHER-NEXT: [[TMP41:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[TMP40]], i64 6
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE12]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE12]]:
+; SINK-GATHER-NEXT: [[TMP42:%.*]] = phi <8 x i32> [ [[TMP36]], %[[PRED_UDIV_CONTINUE10]] ], [ [[TMP41]], %[[PRED_UDIV_IF11]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF13:.*]], label %[[PRED_UDIV_CONTINUE14]]
+; SINK-GATHER: [[PRED_UDIV_IF13]]:
+; SINK-GATHER-NEXT: [[TMP43:%.*]] = extractelement <8 x i64> [[TMP0]], i64 7
+; SINK-GATHER-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP43]]
+; SINK-GATHER-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4
+; SINK-GATHER-NEXT: [[TMP46:%.*]] = udiv i32 [[TMP45]], [[X]]
+; SINK-GATHER-NEXT: [[TMP47:%.*]] = insertelement <8 x i32> [[TMP42]], i32 [[TMP46]], i64 7
+; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE14]]
+; SINK-GATHER: [[PRED_UDIV_CONTINUE14]]:
+; SINK-GATHER-NEXT: [[TMP48:%.*]] = phi <8 x i32> [ [[TMP42]], %[[PRED_UDIV_CONTINUE12]] ], [ [[TMP47]], %[[PRED_UDIV_IF13]] ]
+; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP48]], <8 x i32> [[BROADCAST_SPLAT]]
+; SINK-GATHER-NEXT: [[TMP49]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
; SINK-GATHER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; SINK-GATHER-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i64> [[VEC_IND]], splat (i64 8)
-; SINK-GATHER-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SINK-GATHER-NEXT: br i1 [[TMP67]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; SINK-GATHER: middle.block:
-; SINK-GATHER-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP66]])
+; SINK-GATHER-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SINK-GATHER-NEXT: br i1 [[TMP50]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; SINK-GATHER: [[MIDDLE_BLOCK]]:
+; SINK-GATHER-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP49]])
; SINK-GATHER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; SINK-GATHER-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; SINK-GATHER: scalar.ph:
-; SINK-GATHER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SINK-GATHER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP68]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; SINK-GATHER-NEXT: br label [[FOR_BODY:%.*]]
-; SINK-GATHER: for.body:
-; SINK-GATHER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; SINK-GATHER-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T6:%.*]], [[FOR_INC]] ]
+; SINK-GATHER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SINK-GATHER: [[SCALAR_PH]]:
+; SINK-GATHER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; SINK-GATHER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP51]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; SINK-GATHER-NEXT: br label %[[FOR_BODY:.*]]
+; SINK-GATHER: [[FOR_BODY]]:
+; SINK-GATHER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; SINK-GATHER-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[T6:%.*]], %[[FOR_INC]] ]
; SINK-GATHER-NEXT: [[I7:%.*]] = mul i64 [[I]], 777
-; SINK-GATHER-NEXT: br i1 [[TMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; SINK-GATHER: if.then:
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; SINK-GATHER: [[IF_THEN]]:
; SINK-GATHER-NEXT: [[T0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I7]]
; SINK-GATHER-NEXT: [[T2:%.*]] = load i32, ptr [[T0]], align 4
; SINK-GATHER-NEXT: [[T4:%.*]] = udiv i32 [[T2]], [[X]]
-; SINK-GATHER-NEXT: br label [[FOR_INC]]
-; SINK-GATHER: for.inc:
-; SINK-GATHER-NEXT: [[T5:%.*]] = phi i32 [ [[X]], [[FOR_BODY]] ], [ [[T4]], [[IF_THEN]] ]
+; SINK-GATHER-NEXT: br label %[[FOR_INC]]
+; SINK-GATHER: [[FOR_INC]]:
+; SINK-GATHER-NEXT: [[T5:%.*]] = phi i32 [ [[X]], %[[FOR_BODY]] ], [ [[T4]], %[[IF_THEN]] ]
; SINK-GATHER-NEXT: [[T6]] = add i32 [[R]], [[T5]]
; SINK-GATHER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; SINK-GATHER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; SINK-GATHER-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP4:![0-9]+]]
-; SINK-GATHER: for.end:
-; SINK-GATHER-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], [[FOR_INC]] ], [ [[TMP68]], [[MIDDLE_BLOCK]] ]
+; SINK-GATHER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP4:![0-9]+]]
+; SINK-GATHER: [[FOR_END]]:
+; SINK-GATHER-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], %[[FOR_INC]] ], [ [[TMP51]], %[[MIDDLE_BLOCK]] ]
; SINK-GATHER-NEXT: ret i32 [[T7]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll
index cffe78ba0123a..12cf9dec9472d 100644
--- a/llvm/test/Transforms/LoopVectorize/float-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck --check-prefix VEC4_INTERL1 %s
; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=4 -S | FileCheck --check-prefix VEC4_INTERL2 %s
; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=1 -S | FileCheck --check-prefix VEC1_INTERL2 %s
@@ -18,215 +18,219 @@
define void @fp_iv_loop1_fast_FMF(float %init, ptr noalias nocapture %A, i32 %N) {
-; VEC4_INTERL1-LABEL: @fp_iv_loop1_fast_FMF(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL1: for.body.lr.ph:
+; VEC4_INTERL1-LABEL: define void @fp_iv_loop1_fast_FMF(
+; VEC4_INTERL1-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL1: [[FOR_BODY_LR_PH]]:
; VEC4_INTERL1-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
-; VEC4_INTERL1-NEXT: [[IND_END:%.*]] = fsub fast float [[INIT:%.*]], [[TMP1]]
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = fmul fast <4 x float> [[DOTSPLAT3]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fsub fast <4 x float> [[DOTSPLAT]], [[TMP2]]
-; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], 4.000000e+00
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT4:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT5:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT4]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP4]], align 4
+; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = fsub fast float [[INIT]], [[TMP1]]
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT2]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fsub fast <4 x float> [[BROADCAST_SPLAT]], [[TMP3]]
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], 4.000000e+00
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x float> poison, float [[TMP4]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT3]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP5]], align 4
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fsub fast <4 x float> [[VEC_IND]], [[DOTSPLAT5]]
-; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fsub fast <4 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL1-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL1-NEXT: [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; VEC4_INTERL1: for.end.loopexit:
-; VEC4_INTERL1-NEXT: br label [[FOR_END]]
-; VEC4_INTERL1: for.end:
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VEC4_INTERL1: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL1-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL1: [[FOR_END]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_loop1_fast_FMF(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL2: for.body.lr.ph:
+; VEC4_INTERL2-LABEL: define void @fp_iv_loop1_fast_FMF(
+; VEC4_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL2: [[FOR_BODY_LR_PH]]:
; VEC4_INTERL2-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483640
-; VEC4_INTERL2-NEXT: [[FPINC_INS:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
-; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[FPINC_INS]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
-; VEC4_INTERL2-NEXT: [[IND_END:%.*]] = fsub fast float [[INIT:%.*]], [[TMP1]]
-; VEC4_INTERL2-NEXT: [[DOTSPLAT5:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
-; VEC4_INTERL2-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL2-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fsub fast <4 x float> [[DOTSPLAT3]], [[TMP7]]
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fsub fast <4 x float> [[VEC_IND]], [[DOTSPLAT5]]
-; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
-; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP4]], align 4
-; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP5]], align 4
+; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = fsub fast float [[INIT]], [[TMP1]]
+; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fsub fast <4 x float> [[BROADCAST_SPLAT2]], [[TMP4]]
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fsub fast <4 x float> [[VEC_IND]], [[TMP3]]
+; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 16
+; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP5]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP6]], align 4
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fsub fast <4 x float> [[STEP_ADD]], [[DOTSPLAT5]]
-; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fsub fast <4 x float> [[STEP_ADD]], [[TMP3]]
+; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL3:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL2-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL2-NEXT: [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; VEC4_INTERL2: for.end.loopexit:
-; VEC4_INTERL2-NEXT: br label [[FOR_END]]
-; VEC4_INTERL2: for.end:
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VEC4_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL2: [[FOR_END]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_loop1_fast_FMF(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC1_INTERL2: for.body.lr.ph:
+; VEC1_INTERL2-LABEL: define void @fp_iv_loop1_fast_FMF(
+; VEC1_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC1_INTERL2: [[FOR_BODY_LR_PH]]:
; VEC1_INTERL2-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
-; VEC1_INTERL2-NEXT: [[IND_END:%.*]] = fsub fast float [[INIT:%.*]], [[TMP1]]
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST2]]
+; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = fsub fast float [[INIT]], [[TMP1]]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]]
; VEC1_INTERL2-NEXT: [[OFFSET_IDX:%.*]] = fsub fast float [[INIT]], [[TMP3]]
; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fsub fast float [[OFFSET_IDX]], [[FPINC]]
-; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP9]], i64 4
-; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP8]], align 4
-; VEC1_INTERL2-NEXT: store float [[TMP4]], ptr [[TMP6]], align 4
+; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
+; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP5]], align 4
+; VEC1_INTERL2-NEXT: store float [[TMP4]], ptr [[TMP7]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC1_INTERL2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC1_INTERL2-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC1_INTERL2-NEXT: [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; VEC1_INTERL2: for.end.loopexit:
-; VEC1_INTERL2-NEXT: br label [[FOR_END]]
-; VEC1_INTERL2: for.end:
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VEC1_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC1_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC1_INTERL2: [[FOR_END]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_loop1_fast_FMF(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC2_INTERL1_PRED_STORE: for.body.lr.ph:
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_loop1_fast_FMF(
+; VEC2_INTERL1_PRED_STORE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*:]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY_LR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[IND_END:%.*]] = fsub fast float [[INIT:%.*]], [[TMP1]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT2]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = fmul fast <2 x float> [[DOTSPLAT3]], <float 0.000000e+00, float 1.000000e+00>
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fsub fast <2 x float> [[DOTSPLAT]], [[TMP2]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT4:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT5:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT4]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP4]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = fsub fast float [[INIT]], [[TMP1]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT2]], <float 0.000000e+00, float 1.000000e+00>
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fsub fast <2 x float> [[BROADCAST_SPLAT]], [[TMP3]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP5]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fsub fast <2 x float> [[VEC_IND]], [[DOTSPLAT5]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fsub fast <2 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[FOR_END]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: for.end:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_END]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -263,215 +267,219 @@ define void @fp_iv_loop1_reassoc_FMF(float %init, ptr noalias nocapture %A, i32
;
;
;
-; VEC4_INTERL1-LABEL: @fp_iv_loop1_reassoc_FMF(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL1: for.body.lr.ph:
+; VEC4_INTERL1-LABEL: define void @fp_iv_loop1_reassoc_FMF(
+; VEC4_INTERL1-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL1: [[FOR_BODY_LR_PH]]:
; VEC4_INTERL1-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = fmul reassoc float [[FPINC]], [[DOTCAST]]
-; VEC4_INTERL1-NEXT: [[IND_END:%.*]] = fsub reassoc float [[INIT:%.*]], [[TMP1]]
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = fmul reassoc <4 x float> [[DOTSPLAT3]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fsub reassoc <4 x float> [[DOTSPLAT]], [[TMP2]]
-; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul reassoc float [[FPINC]], 4.000000e+00
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT4:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT5:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT4]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP4]], align 4
+; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = fsub reassoc float [[INIT]], [[TMP1]]
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul reassoc <4 x float> [[BROADCAST_SPLAT2]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fsub reassoc <4 x float> [[BROADCAST_SPLAT]], [[TMP3]]
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = fmul reassoc float [[FPINC]], 4.000000e+00
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x float> poison, float [[TMP4]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT3]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP5]], align 4
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fsub reassoc <4 x float> [[VEC_IND]], [[DOTSPLAT5]]
-; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fsub reassoc <4 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL1-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL1-NEXT: [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; VEC4_INTERL1: for.end.loopexit:
-; VEC4_INTERL1-NEXT: br label [[FOR_END]]
-; VEC4_INTERL1: for.end:
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VEC4_INTERL1: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL1-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL1: [[FOR_END]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_loop1_reassoc_FMF(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL2: for.body.lr.ph:
+; VEC4_INTERL2-LABEL: define void @fp_iv_loop1_reassoc_FMF(
+; VEC4_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL2: [[FOR_BODY_LR_PH]]:
; VEC4_INTERL2-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483640
-; VEC4_INTERL2-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
-; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = fmul reassoc float [[FPINC]], [[DOTCAST]]
-; VEC4_INTERL2-NEXT: [[IND_END:%.*]] = fsub reassoc float [[INIT:%.*]], [[TMP1]]
-; VEC4_INTERL2-NEXT: [[DOTSPLAT5:%.*]] = fmul reassoc <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
-; VEC4_INTERL2-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL2-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = fmul reassoc <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fsub reassoc <4 x float> [[DOTSPLAT3]], [[TMP7]]
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fsub reassoc <4 x float> [[VEC_IND]], [[DOTSPLAT5]]
-; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
-; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP4]], align 4
-; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP5]], align 4
+; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = fsub reassoc float [[INIT]], [[TMP1]]
+; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fmul reassoc <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fmul reassoc <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fsub reassoc <4 x float> [[BROADCAST_SPLAT2]], [[TMP4]]
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fsub reassoc <4 x float> [[VEC_IND]], [[TMP3]]
+; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 16
+; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP5]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP6]], align 4
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fsub reassoc <4 x float> [[STEP_ADD]], [[DOTSPLAT5]]
-; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fsub reassoc <4 x float> [[STEP_ADD]], [[TMP3]]
+; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL3:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL2-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL2-NEXT: [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; VEC4_INTERL2: for.end.loopexit:
-; VEC4_INTERL2-NEXT: br label [[FOR_END]]
-; VEC4_INTERL2: for.end:
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VEC4_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL2: [[FOR_END]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_loop1_reassoc_FMF(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC1_INTERL2: for.body.lr.ph:
+; VEC1_INTERL2-LABEL: define void @fp_iv_loop1_reassoc_FMF(
+; VEC1_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC1_INTERL2: [[FOR_BODY_LR_PH]]:
; VEC1_INTERL2-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = fmul reassoc float [[FPINC]], [[DOTCAST]]
-; VEC1_INTERL2-NEXT: [[IND_END:%.*]] = fsub reassoc float [[INIT:%.*]], [[TMP1]]
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul reassoc float [[FPINC]], [[DOTCAST2]]
+; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = fsub reassoc float [[INIT]], [[TMP1]]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul reassoc float [[FPINC]], [[DOTCAST1]]
; VEC1_INTERL2-NEXT: [[OFFSET_IDX:%.*]] = fsub reassoc float [[INIT]], [[TMP3]]
-; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = fsub reassoc float [[OFFSET_IDX]], [[FPINC]]
-; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP10]], i64 4
-; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP7]], align 4
-; VEC1_INTERL2-NEXT: store float [[TMP6]], ptr [[TMP8]], align 4
+; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fsub reassoc float [[OFFSET_IDX]], [[FPINC]]
+; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
+; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP5]], align 4
+; VEC1_INTERL2-NEXT: store float [[TMP4]], ptr [[TMP7]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC1_INTERL2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC1_INTERL2-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC1_INTERL2-NEXT: [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; VEC1_INTERL2: for.end.loopexit:
-; VEC1_INTERL2-NEXT: br label [[FOR_END]]
-; VEC1_INTERL2: for.end:
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VEC1_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC1_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC1_INTERL2: [[FOR_END]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_loop1_reassoc_FMF(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC2_INTERL1_PRED_STORE: for.body.lr.ph:
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_loop1_reassoc_FMF(
+; VEC2_INTERL1_PRED_STORE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*:]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY_LR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[FPINC:%.*]] = load float, ptr @fp_inc, align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = fmul reassoc float [[FPINC]], [[DOTCAST]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[IND_END:%.*]] = fsub reassoc float [[INIT:%.*]], [[TMP1]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT2]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = fmul reassoc <2 x float> [[DOTSPLAT3]], <float 0.000000e+00, float 1.000000e+00>
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fsub reassoc <2 x float> [[DOTSPLAT]], [[TMP2]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul reassoc float [[FPINC]], 2.000000e+00
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT4:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT5:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT4]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP4]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = fsub reassoc float [[INIT]], [[TMP1]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul reassoc <2 x float> [[BROADCAST_SPLAT2]], <float 0.000000e+00, float 1.000000e+00>
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fsub reassoc <2 x float> [[BROADCAST_SPLAT]], [[TMP3]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = fmul reassoc float [[FPINC]], 2.000000e+00
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP5]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fsub reassoc <2 x float> [[VEC_IND]], [[DOTSPLAT5]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fsub reassoc <2 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[FOR_END]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: for.end:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_END]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -510,195 +518,199 @@ for.end:
define void @fp_iv_loop2(float %init, ptr noalias nocapture %A, i32 %N) #0 {
-; VEC4_INTERL1-LABEL: @fp_iv_loop2(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL1: for.body.preheader:
+; VEC4_INTERL1-LABEL: define void @fp_iv_loop2(
+; VEC4_INTERL1-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL1: [[FOR_BODY_PREHEADER]]:
; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC4_INTERL1-NEXT: [[IND_END:%.*]] = fadd fast float [[INIT:%.*]], [[TMP1]]
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[DOTSPLAT]], <float 0.000000e+00, float 5.000000e-01, float 1.000000e+00, float 1.500000e+00>
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP2]], align 4
+; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 5.000000e-01, float 1.000000e+00, float 1.500000e+00>
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP3]], align 4
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], splat (float 2.000000e+00)
-; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC4_INTERL1-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC4_INTERL1-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ]
; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL1-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL1-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; VEC4_INTERL1: for.end.loopexit:
-; VEC4_INTERL1-NEXT: br label [[FOR_END]]
-; VEC4_INTERL1: for.end:
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VEC4_INTERL1: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL1-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL1: [[FOR_END]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_loop2(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL2: for.body.preheader:
+; VEC4_INTERL2-LABEL: define void @fp_iv_loop2(
+; VEC4_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL2: [[FOR_BODY_PREHEADER]]:
; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483640
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC4_INTERL2-NEXT: [[IND_END:%.*]] = fadd fast float [[INIT:%.*]], [[TMP1]]
-; VEC4_INTERL2-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL2-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[DOTSPLAT]], <float 0.000000e+00, float 5.000000e-01, float 1.000000e+00, float 1.500000e+00>
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 5.000000e-01, float 1.000000e+00, float 1.500000e+00>
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float 2.000000e+00)
-; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
-; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP2]], align 4
-; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP3]], align 4
+; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 16
+; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP3]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP4]], align 4
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], splat (float 4.000000e+00)
-; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC4_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC4_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ]
; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL2-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL2-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; VEC4_INTERL2: for.end.loopexit:
-; VEC4_INTERL2-NEXT: br label [[FOR_END]]
-; VEC4_INTERL2: for.end:
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VEC4_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL2: [[FOR_END]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_loop2(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC1_INTERL2: for.body.preheader:
+; VEC1_INTERL2-LABEL: define void @fp_iv_loop2(
+; VEC1_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC1_INTERL2: [[FOR_BODY_PREHEADER]]:
; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC1_INTERL2-NEXT: [[IND_END:%.*]] = fadd fast float [[INIT:%.*]], [[TMP1]]
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[DOTCAST2]], 5.000000e-01
+; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[DOTCAST1]], 5.000000e-01
; VEC1_INTERL2-NEXT: [[OFFSET_IDX:%.*]] = fadd fast float [[INIT]], [[TMP3]]
; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fadd fast float [[OFFSET_IDX]], 5.000000e-01
-; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP9]], i64 4
-; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP8]], align 4
-; VEC1_INTERL2-NEXT: store float [[TMP4]], ptr [[TMP6]], align 4
+; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
+; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP5]], align 4
+; VEC1_INTERL2-NEXT: store float [[TMP4]], ptr [[TMP7]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC1_INTERL2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_PREHEADER]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC1_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_PREHEADER]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC1_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC1_INTERL2-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC1_INTERL2-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; VEC1_INTERL2: for.end.loopexit:
-; VEC1_INTERL2-NEXT: br label [[FOR_END]]
-; VEC1_INTERL2: for.end:
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VEC1_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC1_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC1_INTERL2: [[FOR_END]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_loop2(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC2_INTERL1_PRED_STORE: for.body.preheader:
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_loop2(
+; VEC2_INTERL1_PRED_STORE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*:]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY_PREHEADER]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC2_INTERL1_PRED_STORE-NEXT: [[IND_END:%.*]] = fadd fast float [[INIT:%.*]], [[TMP1]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fadd fast <2 x float> [[DOTSPLAT]], <float 0.000000e+00, float 5.000000e-01>
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP2]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fadd fast <2 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 5.000000e-01>
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP3]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fadd fast <2 x float> [[VEC_IND]], splat (float 1.000000e+00)
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_PREHEADER]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[FOR_END]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_PREHEADER]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: for.end:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_END]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -741,65 +753,66 @@ for.end:
define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias nocapture %B, ptr noalias nocapture %C, i32 %N) {
-; VEC4_INTERL1-LABEL: @fp_iv_loop3(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL1-NEXT: br i1 [[CMP9]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL1: for.body.lr.ph:
+; VEC4_INTERL1-LABEL: define void @fp_iv_loop3(
+; VEC4_INTERL1-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL1-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL1-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL1: [[FOR_BODY_LR_PH]]:
; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = load float, ptr @fp_inc, align 4
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], 2147483644
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = fmul fast float [[DOTCAST]], -5.000000e-01
-; VEC4_INTERL1-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
-; VEC4_INTERL1-NEXT: [[DOTCAST2:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP0]], [[DOTCAST2]]
-; VEC4_INTERL1-NEXT: [[IND_END3:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT5:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT6:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT5]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; VEC4_INTERL1-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: [[TMP13:%.*]] = fmul fast <4 x float> [[DOTSPLAT3]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[DOTSPLAT6]], [[TMP13]]
-; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = fmul fast float [[TMP0]], 4.000000e+00
-; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
+; VEC4_INTERL1-NEXT: [[DOTCAST1:%.*]] = uitofp nneg i64 [[N_VEC]] to float
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = fmul fast float [[TMP0]], [[DOTCAST1]]
+; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 0x3FB99999A0000000, float 0xBFD99999A0000000, float 0xBFECCCCCC0000000, float 0xBFF6666660000000>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND9:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND9]], ptr [[TMP6]], align 4
-; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[VEC_IND9]], [[DOTSPLAT]]
-; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float -5.000000e-01)
-; VEC4_INTERL1-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[TMP8]], [[TMP7]]
-; VEC4_INTERL1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: store <4 x float> [[TMP9]], ptr [[TMP10]], align 4
-; VEC4_INTERL1-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i8], ptr [[C:%.*]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: store <4 x float> [[TMP8]], ptr [[TMP11]], align 4
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT4]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT5]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL1-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT3]], [[TMP6]]
+; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = fmul fast float [[TMP0]], 4.000000e+00
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x float> poison, float [[TMP7]], i64 0
+; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT6]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 0x3FB99999A0000000, float 0xBFD99999A0000000, float 0xBFECCCCCC0000000, float 0xBFF6666660000000>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND8:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND8]], ptr [[TMP8]], align 4
+; VEC4_INTERL1-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[VEC_IND8]], [[BROADCAST_SPLAT]]
+; VEC4_INTERL1-NEXT: [[TMP10:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float -5.000000e-01)
+; VEC4_INTERL1-NEXT: [[TMP11:%.*]] = fadd fast <4 x float> [[TMP10]], [[TMP9]]
+; VEC4_INTERL1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: store <4 x float> [[TMP11]], ptr [[TMP12]], align 4
+; VEC4_INTERL1-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: store <4 x float> [[TMP10]], ptr [[TMP13]], align 4
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], splat (float -2.000000e+00)
-; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT10]] = fadd fast <4 x float> [[VEC_IND9]], [[BROADCAST_SPLAT]]
-; VEC4_INTERL1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT9]] = fadd fast <4 x float> [[VEC_IND8]], [[BROADCAST_SPLAT7]]
+; VEC4_INTERL1-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL1-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP1]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL4:%.*]] = phi float [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[CONV1:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL10:%.*]] = phi float [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL11:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL10]], %[[SCALAR_PH]] ], [ [[CONV1:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL11]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL1-NEXT: store float [[X_011]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL1-NEXT: [[ADD]] = fadd fast float [[X_011]], [[TMP0]]
@@ -812,79 +825,80 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; VEC4_INTERL1: for.end.loopexit:
-; VEC4_INTERL1-NEXT: br label [[FOR_END]]
-; VEC4_INTERL1: for.end:
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VEC4_INTERL1: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL1-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL1: [[FOR_END]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_loop3(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL2-NEXT: br i1 [[CMP9]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL2: for.body.lr.ph:
+; VEC4_INTERL2-LABEL: define void @fp_iv_loop3(
+; VEC4_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL2-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL2-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL2: [[FOR_BODY_LR_PH]]:
; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = load float, ptr @fp_inc, align 4
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], 2147483640
-; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = fmul fast float [[DOTCAST]], -5.000000e-01
-; VEC4_INTERL2-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
-; VEC4_INTERL2-NEXT: [[DOTCAST2:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP0]], [[DOTCAST2]]
-; VEC4_INTERL2-NEXT: [[IND_END3:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
-; VEC4_INTERL2-NEXT: [[DOTSPLAT5:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
-; VEC4_INTERL2-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; VEC4_INTERL2-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL2-NEXT: [[DOTSPLATINSERT6:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
-; VEC4_INTERL2-NEXT: [[DOTSPLAT7:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT6]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL2-NEXT: [[TMP19:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[DOTSPLAT7]], [[TMP19]]
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 0x3FB99999A0000000, float 0xBFD99999A0000000, float 0xBFECCCCCC0000000, float 0xBFF6666660000000>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND10:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT13:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[STEP_ADD11:%.*]] = fadd fast <4 x float> [[VEC_IND10]], [[DOTSPLAT5]]
-; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 16
-; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND10]], ptr [[TMP6]], align 4
-; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD11]], ptr [[TMP7]], align 4
-; VEC4_INTERL2-NEXT: [[TMP8:%.*]] = fadd fast <4 x float> [[VEC_IND10]], [[DOTSPLAT]]
-; VEC4_INTERL2-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[STEP_ADD11]], [[DOTSPLAT]]
-; VEC4_INTERL2-NEXT: [[TMP10:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float -5.000000e-01)
-; VEC4_INTERL2-NEXT: [[TMP11:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float -2.500000e+00)
-; VEC4_INTERL2-NEXT: [[TMP12:%.*]] = fadd fast <4 x float> [[TMP10]], [[TMP8]]
-; VEC4_INTERL2-NEXT: [[TMP13:%.*]] = fadd fast <4 x float> [[TMP11]], [[TMP9]]
-; VEC4_INTERL2-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 16
-; VEC4_INTERL2-NEXT: store <4 x float> [[TMP12]], ptr [[TMP14]], align 4
-; VEC4_INTERL2-NEXT: store <4 x float> [[TMP13]], ptr [[TMP15]], align 4
-; VEC4_INTERL2-NEXT: [[TMP16:%.*]] = getelementptr inbounds [4 x i8], ptr [[C:%.*]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
+; VEC4_INTERL2-NEXT: [[DOTCAST1:%.*]] = uitofp nneg i64 [[N_VEC]] to float
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fmul fast float [[TMP0]], [[DOTCAST1]]
+; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT4]], <4 x float> poison, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT5]], [[TMP7]]
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 0x3FB99999A0000000, float 0xBFD99999A0000000, float 0xBFECCCCCC0000000, float 0xBFF6666660000000>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND6:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[STEP_ADD7:%.*]] = fadd fast <4 x float> [[VEC_IND6]], [[TMP6]]
+; VEC4_INTERL2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
+; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND6]], ptr [[TMP8]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD7]], ptr [[TMP9]], align 4
+; VEC4_INTERL2-NEXT: [[TMP10:%.*]] = fadd fast <4 x float> [[VEC_IND6]], [[BROADCAST_SPLAT3]]
+; VEC4_INTERL2-NEXT: [[TMP11:%.*]] = fadd fast <4 x float> [[STEP_ADD7]], [[BROADCAST_SPLAT3]]
+; VEC4_INTERL2-NEXT: [[TMP12:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float -5.000000e-01)
+; VEC4_INTERL2-NEXT: [[TMP13:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float -2.500000e+00)
+; VEC4_INTERL2-NEXT: [[TMP14:%.*]] = fadd fast <4 x float> [[TMP12]], [[TMP10]]
+; VEC4_INTERL2-NEXT: [[TMP15:%.*]] = fadd fast <4 x float> [[TMP13]], [[TMP11]]
+; VEC4_INTERL2-NEXT: [[TMP16:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
; VEC4_INTERL2-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 16
-; VEC4_INTERL2-NEXT: store <4 x float> [[TMP10]], ptr [[TMP16]], align 4
-; VEC4_INTERL2-NEXT: store <4 x float> [[TMP11]], ptr [[TMP17]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[TMP14]], ptr [[TMP16]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[TMP15]], ptr [[TMP17]], align 4
+; VEC4_INTERL2-NEXT: [[TMP18:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP18]], i64 16
+; VEC4_INTERL2-NEXT: store <4 x float> [[TMP12]], ptr [[TMP18]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[TMP13]], ptr [[TMP19]], align 4
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], splat (float -4.000000e+00)
-; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT13]] = fadd fast <4 x float> [[STEP_ADD11]], [[DOTSPLAT5]]
-; VEC4_INTERL2-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT8]] = fadd fast <4 x float> [[STEP_ADD7]], [[TMP6]]
+; VEC4_INTERL2-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL2-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP1]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL4:%.*]] = phi float [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[CONV1:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL9:%.*]] = phi float [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL10:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL9]], %[[SCALAR_PH]] ], [ [[CONV1:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL10]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL2-NEXT: store float [[X_011]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL2-NEXT: [[ADD]] = fadd fast float [[X_011]], [[TMP0]]
@@ -897,72 +911,73 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; VEC4_INTERL2: for.end.loopexit:
-; VEC4_INTERL2-NEXT: br label [[FOR_END]]
-; VEC4_INTERL2: for.end:
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VEC4_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL2: [[FOR_END]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_loop3(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC1_INTERL2-NEXT: br i1 [[CMP9]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC1_INTERL2: for.body.lr.ph:
+; VEC1_INTERL2-LABEL: define void @fp_iv_loop3(
+; VEC1_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC1_INTERL2-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; VEC1_INTERL2-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC1_INTERL2: [[FOR_BODY_LR_PH]]:
; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = load float, ptr @fp_inc, align 4
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = zext nneg i32 [[N]] to i64
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], 2147483646
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = fmul fast float [[DOTCAST]], -5.000000e-01
-; VEC1_INTERL2-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
-; VEC1_INTERL2-NEXT: [[DOTCAST2:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP0]], [[DOTCAST2]]
-; VEC1_INTERL2-NEXT: [[IND_END3:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 1
-; VEC1_INTERL2-NEXT: [[DOTCAST5:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = fmul fast float [[DOTCAST5]], -5.000000e-01
-; VEC1_INTERL2-NEXT: [[DOTCAST6:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP0]], [[DOTCAST6]]
-; VEC1_INTERL2-NEXT: [[OFFSET_IDX7:%.*]] = fadd fast float [[INIT]], [[TMP6]]
-; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fadd fast float [[OFFSET_IDX7]], [[TMP0]]
-; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP4]]
-; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX7]], ptr [[TMP8]], align 4
-; VEC1_INTERL2-NEXT: store float [[TMP7]], ptr [[TMP9]], align 4
-; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = fadd fast float [[OFFSET_IDX7]], [[TMP0]]
-; VEC1_INTERL2-NEXT: [[TMP11:%.*]] = fadd fast float [[TMP7]], [[TMP0]]
-; VEC1_INTERL2-NEXT: [[TMP12:%.*]] = fadd fast float [[TMP5]], 0xBFD99999A0000000
-; VEC1_INTERL2-NEXT: [[TMP13:%.*]] = fadd fast float [[TMP5]], 0xBFECCCCCC0000000
-; VEC1_INTERL2-NEXT: [[TMP14:%.*]] = fadd fast float [[TMP12]], [[TMP10]]
-; VEC1_INTERL2-NEXT: [[TMP15:%.*]] = fadd fast float [[TMP13]], [[TMP11]]
-; VEC1_INTERL2-NEXT: [[TMP16:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP17:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP4]]
-; VEC1_INTERL2-NEXT: store float [[TMP14]], ptr [[TMP16]], align 4
-; VEC1_INTERL2-NEXT: store float [[TMP15]], ptr [[TMP17]], align 4
-; VEC1_INTERL2-NEXT: [[TMP18:%.*]] = getelementptr inbounds [4 x i8], ptr [[C:%.*]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[TMP4]]
-; VEC1_INTERL2-NEXT: store float [[TMP12]], ptr [[TMP18]], align 4
-; VEC1_INTERL2-NEXT: store float [[TMP13]], ptr [[TMP19]], align 4
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
+; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = uitofp nneg i64 [[N_VEC]] to float
+; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fmul fast float [[TMP0]], [[DOTCAST1]]
+; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = or disjoint i64 [[INDEX]], 1
+; VEC1_INTERL2-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast float [[DOTCAST2]], -5.000000e-01
+; VEC1_INTERL2-NEXT: [[DOTCAST3:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP0]], [[DOTCAST3]]
+; VEC1_INTERL2-NEXT: [[OFFSET_IDX4:%.*]] = fadd fast float [[INIT]], [[TMP8]]
+; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = fadd fast float [[OFFSET_IDX4]], [[TMP0]]
+; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP6]]
+; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX4]], ptr [[TMP10]], align 4
+; VEC1_INTERL2-NEXT: store float [[TMP9]], ptr [[TMP11]], align 4
+; VEC1_INTERL2-NEXT: [[TMP12:%.*]] = fadd fast float [[OFFSET_IDX4]], [[TMP0]]
+; VEC1_INTERL2-NEXT: [[TMP13:%.*]] = fadd fast float [[TMP9]], [[TMP0]]
+; VEC1_INTERL2-NEXT: [[TMP14:%.*]] = fadd fast float [[TMP7]], 0xBFD99999A0000000
+; VEC1_INTERL2-NEXT: [[TMP15:%.*]] = fadd fast float [[TMP7]], 0xBFECCCCCC0000000
+; VEC1_INTERL2-NEXT: [[TMP16:%.*]] = fadd fast float [[TMP14]], [[TMP12]]
+; VEC1_INTERL2-NEXT: [[TMP17:%.*]] = fadd fast float [[TMP15]], [[TMP13]]
+; VEC1_INTERL2-NEXT: [[TMP18:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP6]]
+; VEC1_INTERL2-NEXT: store float [[TMP16]], ptr [[TMP18]], align 4
+; VEC1_INTERL2-NEXT: store float [[TMP17]], ptr [[TMP19]], align 4
+; VEC1_INTERL2-NEXT: [[TMP20:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP21:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[TMP6]]
+; VEC1_INTERL2-NEXT: store float [[TMP14]], ptr [[TMP20]], align 4
+; VEC1_INTERL2-NEXT: store float [[TMP15]], ptr [[TMP21]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC1_INTERL2-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC1_INTERL2-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP1]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, [[FOR_BODY_LR_PH]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL4:%.*]] = phi float [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[CONV1:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, %[[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL6:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[CONV1:%.*]], %[[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC1_INTERL2-NEXT: store float [[X_011]], ptr [[ARRAYIDX]], align 4
; VEC1_INTERL2-NEXT: [[ADD]] = fadd fast float [[X_011]], [[TMP0]]
@@ -975,66 +990,67 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; VEC1_INTERL2: for.end.loopexit:
-; VEC1_INTERL2-NEXT: br label [[FOR_END]]
-; VEC1_INTERL2: for.end:
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VEC1_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC1_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC1_INTERL2: [[FOR_END]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_loop3(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP9]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; VEC2_INTERL1_PRED_STORE: for.body.lr.ph:
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_loop3(
+; VEC2_INTERL1_PRED_STORE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*:]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 0
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END:.*]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY_LR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = load float, ptr @fp_inc, align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = zext nneg i32 [[N]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], 2147483646
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = fmul fast float [[DOTCAST]], -5.000000e-01
-; VEC2_INTERL1_PRED_STORE-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST2:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul fast float [[TMP0]], [[DOTCAST2]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[IND_END3:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT5:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT6:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT5]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT2]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP13:%.*]] = fmul fast <2 x float> [[DOTSPLAT3]], <float 0.000000e+00, float 1.000000e+00>
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fadd fast <2 x float> [[DOTSPLAT6]], [[TMP13]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = fmul fast float [[TMP0]], 2.000000e+00
-; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fadd fast float [[TMP2]], 0x3FB99999A0000000
+; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST1:%.*]] = uitofp nneg i64 [[N_VEC]] to float
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = fmul fast float [[TMP0]], [[DOTCAST1]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ <float 0x3FB99999A0000000, float 0xBFD99999A0000000>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND9:%.*]] = phi <2 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND9]], ptr [[TMP6]], align 4
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP7:%.*]] = fadd fast <2 x float> [[VEC_IND9]], [[DOTSPLAT]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP8:%.*]] = fadd fast <2 x float> [[VEC_IND]], splat (float -5.000000e-01)
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP9:%.*]] = fadd fast <2 x float> [[TMP8]], [[TMP7]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[TMP9]], ptr [[TMP10]], align 4
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i8], ptr [[C:%.*]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[TMP8]], ptr [[TMP11]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT2]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT4]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT5]], <float 0.000000e+00, float 1.000000e+00>
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDUCTION:%.*]] = fadd fast <2 x float> [[BROADCAST_SPLAT3]], [[TMP6]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP7:%.*]] = fmul fast float [[TMP0]], 2.000000e+00
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <2 x float> poison, float [[TMP7]], i64 0
+; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT6]], <2 x float> poison, <2 x i32> zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ <float 0x3FB99999A0000000, float 0xBFD99999A0000000>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND8:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND8]], ptr [[TMP8]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP9:%.*]] = fadd fast <2 x float> [[VEC_IND8]], [[BROADCAST_SPLAT]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP10:%.*]] = fadd fast <2 x float> [[VEC_IND]], splat (float -5.000000e-01)
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP11:%.*]] = fadd fast <2 x float> [[TMP10]], [[TMP9]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[TMP11]], ptr [[TMP12]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[TMP10]], ptr [[TMP13]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fadd fast <2 x float> [[VEC_IND]], splat (float -1.000000e+00)
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT10]] = fadd fast <2 x float> [[VEC_IND9]], [[BROADCAST_SPLAT]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT9]] = fadd fast <2 x float> [[VEC_IND8]], [[BROADCAST_SPLAT7]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP1]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[Y_012:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, [[FOR_BODY_LR_PH]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_011:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[FOR_END]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[Y_012:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0x3FB99999A0000000, %[[FOR_BODY_LR_PH]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_011:%.*]] = phi float [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[FOR_BODY_LR_PH]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[X_011]], ptr [[ARRAYIDX]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[ADD]] = fadd fast float [[X_011]], [[TMP0]]
@@ -1047,8 +1063,8 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: for.end:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_END]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -1095,186 +1111,190 @@ for.end:
define void @fp_iv_loop4(ptr noalias nocapture %A, i32 %N) {
-; VEC4_INTERL1-LABEL: @fp_iv_loop4(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL1: for.body.preheader:
+; VEC4_INTERL1-LABEL: define void @fp_iv_loop4(
+; VEC4_INTERL1-SAME: ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL1-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL1-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL1: [[FOR_BODY_PREHEADER]]:
; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC4_INTERL1-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP2]], align 4
+; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP3]], align 4
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], splat (float 2.000000e+00)
-; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC4_INTERL1-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 1.000000e+00, %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC4_INTERL1-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ]
; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL1-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL1-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; VEC4_INTERL1: for.end.loopexit:
-; VEC4_INTERL1-NEXT: br label [[FOR_END]]
-; VEC4_INTERL1: for.end:
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VEC4_INTERL1: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL1-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL1: [[FOR_END]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_loop4(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC4_INTERL2: for.body.preheader:
+; VEC4_INTERL2-LABEL: define void @fp_iv_loop4(
+; VEC4_INTERL2-SAME: ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC4_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC4_INTERL2: [[FOR_BODY_PREHEADER]]:
; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483640
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC4_INTERL2-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float 2.000000e+00)
-; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
-; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP2]], align 4
-; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP3]], align 4
+; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 16
+; VEC4_INTERL2-NEXT: store <4 x float> [[VEC_IND]], ptr [[TMP3]], align 4
+; VEC4_INTERL2-NEXT: store <4 x float> [[STEP_ADD]], ptr [[TMP4]], align 4
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], splat (float 4.000000e+00)
-; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[FOR_BODY_PREHEADER]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC4_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 1.000000e+00, %[[FOR_BODY_PREHEADER]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC4_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ]
; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC4_INTERL2-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL2-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; VEC4_INTERL2: for.end.loopexit:
-; VEC4_INTERL2-NEXT: br label [[FOR_END]]
-; VEC4_INTERL2: for.end:
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VEC4_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC4_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC4_INTERL2: [[FOR_END]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_loop4(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC1_INTERL2: for.body.preheader:
+; VEC1_INTERL2-LABEL: define void @fp_iv_loop4(
+; VEC1_INTERL2-SAME: ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC1_INTERL2-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC1_INTERL2-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC1_INTERL2: [[FOR_BODY_PREHEADER]]:
; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC1_INTERL2-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[DOTCAST2]], 5.000000e-01
+; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[DOTCAST1]], 5.000000e-01
; VEC1_INTERL2-NEXT: [[OFFSET_IDX:%.*]] = fadd fast float [[TMP3]], 1.000000e+00
; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fadd fast float [[TMP3]], 1.500000e+00
-; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP9]], i64 4
-; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP8]], align 4
-; VEC1_INTERL2-NEXT: store float [[TMP4]], ptr [[TMP6]], align 4
+; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
+; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP5]], align 4
+; VEC1_INTERL2-NEXT: store float [[TMP4]], ptr [[TMP7]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC1_INTERL2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[FOR_BODY_PREHEADER]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC1_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 1.000000e+00, %[[FOR_BODY_PREHEADER]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC1_INTERL2-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC1_INTERL2-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC1_INTERL2-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; VEC1_INTERL2: for.end.loopexit:
-; VEC1_INTERL2-NEXT: br label [[FOR_END]]
-; VEC1_INTERL2: for.end:
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VEC1_INTERL2: [[FOR_END_LOOPEXIT]]:
+; VEC1_INTERL2-NEXT: br label %[[FOR_END]]
+; VEC1_INTERL2: [[FOR_END]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_loop4(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; VEC2_INTERL1_PRED_STORE: for.body.preheader:
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_loop4(
+; VEC2_INTERL1_PRED_STORE-SAME: ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*:]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY_PREHEADER]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[N]], 1
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483646
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = fmul fast float [[DOTCAST]], 5.000000e-01
-; VEC2_INTERL1_PRED_STORE-NEXT: [[IND_END:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ <float 1.000000e+00, float 1.500000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP2]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ <float 1.000000e+00, float 1.500000e+00>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP3]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fadd fast <2 x float> [[VEC_IND]], splat (float 1.000000e+00)
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[FOR_BODY_PREHEADER]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[FOR_END]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 1.000000e+00, %[[FOR_BODY_PREHEADER]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: for.end:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_END]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -1304,307 +1324,311 @@ for.end:
define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
-; VEC4_INTERL1-LABEL: @non_primary_iv_float_scalar(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; VEC4_INTERL1-LABEL: define void @non_primary_iv_float_scalar(
+; VEC4_INTERL1-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL1-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE7:%.*]] ]
-; VEC4_INTERL1-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
+; VEC4_INTERL1-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
; VEC4_INTERL1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], zeroinitializer
; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
-; VEC4_INTERL1-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; VEC4_INTERL1: pred.store.if:
-; VEC4_INTERL1-NEXT: store float [[DOTCAST2]], ptr [[TMP0]], align 4
-; VEC4_INTERL1-NEXT: br label [[PRED_STORE_CONTINUE]]
-; VEC4_INTERL1: pred.store.continue:
-; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; VEC4_INTERL1-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3:%.*]]
-; VEC4_INTERL1: pred.store.if2:
-; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 4
-; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
-; VEC4_INTERL1-NEXT: store float [[TMP7]], ptr [[TMP6]], align 4
-; VEC4_INTERL1-NEXT: br label [[PRED_STORE_CONTINUE3]]
-; VEC4_INTERL1: pred.store.continue3:
-; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; VEC4_INTERL1-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
-; VEC4_INTERL1: pred.store.if4:
-; VEC4_INTERL1-NEXT: [[TMP9:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 8
-; VEC4_INTERL1-NEXT: [[TMP11:%.*]] = fadd fast float [[DOTCAST2]], 2.000000e+00
-; VEC4_INTERL1-NEXT: store float [[TMP11]], ptr [[TMP10]], align 4
-; VEC4_INTERL1-NEXT: br label [[PRED_STORE_CONTINUE5]]
-; VEC4_INTERL1: pred.store.continue5:
-; VEC4_INTERL1-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; VEC4_INTERL1-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7]]
-; VEC4_INTERL1: pred.store.if6:
-; VEC4_INTERL1-NEXT: [[TMP13:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL1-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i64 12
-; VEC4_INTERL1-NEXT: [[TMP15:%.*]] = fadd fast float [[DOTCAST2]], 3.000000e+00
-; VEC4_INTERL1-NEXT: store float [[TMP15]], ptr [[TMP14]], align 4
-; VEC4_INTERL1-NEXT: br label [[PRED_STORE_CONTINUE7]]
-; VEC4_INTERL1: pred.store.continue7:
+; VEC4_INTERL1-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC4_INTERL1: [[PRED_STORE_IF]]:
+; VEC4_INTERL1-NEXT: store float [[DOTCAST1]], ptr [[TMP0]], align 4
+; VEC4_INTERL1-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC4_INTERL1: [[PRED_STORE_CONTINUE]]:
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; VEC4_INTERL1-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VEC4_INTERL1: [[PRED_STORE_IF2]]:
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i64 4
+; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = fadd fast float [[DOTCAST1]], 1.000000e+00
+; VEC4_INTERL1-NEXT: store float [[TMP6]], ptr [[TMP5]], align 4
+; VEC4_INTERL1-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; VEC4_INTERL1: [[PRED_STORE_CONTINUE3]]:
+; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; VEC4_INTERL1-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VEC4_INTERL1: [[PRED_STORE_IF4]]:
+; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 8
+; VEC4_INTERL1-NEXT: [[TMP10:%.*]] = fadd fast float [[DOTCAST1]], 2.000000e+00
+; VEC4_INTERL1-NEXT: store float [[TMP10]], ptr [[TMP9]], align 4
+; VEC4_INTERL1-NEXT: br label %[[PRED_STORE_CONTINUE5]]
+; VEC4_INTERL1: [[PRED_STORE_CONTINUE5]]:
+; VEC4_INTERL1-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; VEC4_INTERL1-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
+; VEC4_INTERL1: [[PRED_STORE_IF6]]:
+; VEC4_INTERL1-NEXT: [[TMP12:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL1-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 12
+; VEC4_INTERL1-NEXT: [[TMP14:%.*]] = fadd fast float [[DOTCAST1]], 3.000000e+00
+; VEC4_INTERL1-NEXT: store float [[TMP14]], ptr [[TMP13]], align 4
+; VEC4_INTERL1-NEXT: br label %[[PRED_STORE_CONTINUE7]]
+; VEC4_INTERL1: [[PRED_STORE_CONTINUE7]]:
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; VEC4_INTERL1-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL1-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi float [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC4_INTERL1-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL8:%.*]] = phi float [ [[DOTCAST]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC4_INTERL1-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], %[[FOR_INC]] ], [ [[BC_RESUME_VAL8]], %[[SCALAR_PH]] ]
; VEC4_INTERL1-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[I]]
; VEC4_INTERL1-NEXT: [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
; VEC4_INTERL1-NEXT: [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
-; VEC4_INTERL1-NEXT: br i1 [[VAR2]], label [[IF_PRED:%.*]], label [[FOR_INC]]
-; VEC4_INTERL1: if.pred:
+; VEC4_INTERL1-NEXT: br i1 [[VAR2]], label %[[IF_PRED:.*]], label %[[FOR_INC]]
+; VEC4_INTERL1: [[IF_PRED]]:
; VEC4_INTERL1-NEXT: store float [[J]], ptr [[VAR0]], align 4
-; VEC4_INTERL1-NEXT: br label [[FOR_INC]]
-; VEC4_INTERL1: for.inc:
+; VEC4_INTERL1-NEXT: br label %[[FOR_INC]]
+; VEC4_INTERL1: [[FOR_INC]]:
; VEC4_INTERL1-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; VEC4_INTERL1-NEXT: [[J_NEXT]] = fadd fast float [[J]], 1.000000e+00
; VEC4_INTERL1-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; VEC4_INTERL1-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
-; VEC4_INTERL1: for.end:
+; VEC4_INTERL1-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
+; VEC4_INTERL1: [[FOR_END]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @non_primary_iv_float_scalar(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; VEC4_INTERL2-LABEL: define void @non_primary_iv_float_scalar(
+; VEC4_INTERL2-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL2-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775800
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE16:%.*]] ]
-; VEC4_INTERL2-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; VEC4_INTERL2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; VEC4_INTERL2-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
-; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], zeroinitializer
-; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD3]], zeroinitializer
-; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
-; VEC4_INTERL2-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; VEC4_INTERL2: pred.store.if:
-; VEC4_INTERL2-NEXT: store float [[DOTCAST2]], ptr [[TMP1]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE]]
-; VEC4_INTERL2: pred.store.continue:
-; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
-; VEC4_INTERL2-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
-; VEC4_INTERL2: pred.store.if3:
-; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 4
-; VEC4_INTERL2-NEXT: [[TMP9:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
-; VEC4_INTERL2-NEXT: store float [[TMP9]], ptr [[TMP8]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; VEC4_INTERL2: pred.store.continue4:
-; VEC4_INTERL2-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
-; VEC4_INTERL2-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
-; VEC4_INTERL2: pred.store.if5:
-; VEC4_INTERL2-NEXT: [[TMP11:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 8
-; VEC4_INTERL2-NEXT: [[TMP13:%.*]] = fadd fast float [[DOTCAST2]], 2.000000e+00
-; VEC4_INTERL2-NEXT: store float [[TMP13]], ptr [[TMP12]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; VEC4_INTERL2: pred.store.continue6:
-; VEC4_INTERL2-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
-; VEC4_INTERL2-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; VEC4_INTERL2: pred.store.if7:
-; VEC4_INTERL2-NEXT: [[TMP15:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 12
-; VEC4_INTERL2-NEXT: [[TMP17:%.*]] = fadd fast float [[DOTCAST2]], 3.000000e+00
-; VEC4_INTERL2-NEXT: store float [[TMP17]], ptr [[TMP16]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; VEC4_INTERL2: pred.store.continue8:
-; VEC4_INTERL2-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
-; VEC4_INTERL2-NEXT: br i1 [[TMP18]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; VEC4_INTERL2: pred.store.if9:
-; VEC4_INTERL2-NEXT: [[TMP19:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP19]], i64 16
-; VEC4_INTERL2-NEXT: [[TMP21:%.*]] = fadd fast float [[DOTCAST2]], 4.000000e+00
-; VEC4_INTERL2-NEXT: store float [[TMP21]], ptr [[TMP20]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; VEC4_INTERL2: pred.store.continue10:
-; VEC4_INTERL2-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1
-; VEC4_INTERL2-NEXT: br i1 [[TMP22]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]
-; VEC4_INTERL2: pred.store.if11:
-; VEC4_INTERL2-NEXT: [[TMP23:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP23]], i64 20
-; VEC4_INTERL2-NEXT: [[TMP25:%.*]] = fadd fast float [[DOTCAST2]], 5.000000e+00
-; VEC4_INTERL2-NEXT: store float [[TMP25]], ptr [[TMP24]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; VEC4_INTERL2: pred.store.continue12:
-; VEC4_INTERL2-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2
-; VEC4_INTERL2-NEXT: br i1 [[TMP26]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]]
-; VEC4_INTERL2: pred.store.if13:
-; VEC4_INTERL2-NEXT: [[TMP27:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i64 24
-; VEC4_INTERL2-NEXT: [[TMP29:%.*]] = fadd fast float [[DOTCAST2]], 6.000000e+00
-; VEC4_INTERL2-NEXT: store float [[TMP29]], ptr [[TMP28]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE14]]
-; VEC4_INTERL2: pred.store.continue14:
-; VEC4_INTERL2-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3
-; VEC4_INTERL2-NEXT: br i1 [[TMP30]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16]]
-; VEC4_INTERL2: pred.store.if15:
-; VEC4_INTERL2-NEXT: [[TMP31:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP31]], i64 28
-; VEC4_INTERL2-NEXT: [[TMP33:%.*]] = fadd fast float [[DOTCAST2]], 7.000000e+00
-; VEC4_INTERL2-NEXT: store float [[TMP33]], ptr [[TMP32]], align 4
-; VEC4_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE16]]
-; VEC4_INTERL2: pred.store.continue16:
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE16:.*]] ]
+; VEC4_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; VEC4_INTERL2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; VEC4_INTERL2-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], zeroinitializer
+; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD2]], zeroinitializer
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; VEC4_INTERL2-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC4_INTERL2: [[PRED_STORE_IF]]:
+; VEC4_INTERL2-NEXT: store float [[DOTCAST1]], ptr [[TMP0]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE]]:
+; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; VEC4_INTERL2-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VEC4_INTERL2: [[PRED_STORE_IF3]]:
+; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
+; VEC4_INTERL2-NEXT: [[TMP8:%.*]] = fadd fast float [[DOTCAST1]], 1.000000e+00
+; VEC4_INTERL2-NEXT: store float [[TMP8]], ptr [[TMP7]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE4]]:
+; VEC4_INTERL2-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; VEC4_INTERL2-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VEC4_INTERL2: [[PRED_STORE_IF5]]:
+; VEC4_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 8
+; VEC4_INTERL2-NEXT: [[TMP12:%.*]] = fadd fast float [[DOTCAST1]], 2.000000e+00
+; VEC4_INTERL2-NEXT: store float [[TMP12]], ptr [[TMP11]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE6]]:
+; VEC4_INTERL2-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; VEC4_INTERL2-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; VEC4_INTERL2: [[PRED_STORE_IF7]]:
+; VEC4_INTERL2-NEXT: [[TMP14:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[TMP14]], i64 12
+; VEC4_INTERL2-NEXT: [[TMP16:%.*]] = fadd fast float [[DOTCAST1]], 3.000000e+00
+; VEC4_INTERL2-NEXT: store float [[TMP16]], ptr [[TMP15]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE8]]:
+; VEC4_INTERL2-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
+; VEC4_INTERL2-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; VEC4_INTERL2: [[PRED_STORE_IF9]]:
+; VEC4_INTERL2-NEXT: [[TMP18:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP18]], i64 16
+; VEC4_INTERL2-NEXT: [[TMP20:%.*]] = fadd fast float [[DOTCAST1]], 4.000000e+00
+; VEC4_INTERL2-NEXT: store float [[TMP20]], ptr [[TMP19]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE10]]:
+; VEC4_INTERL2-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
+; VEC4_INTERL2-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; VEC4_INTERL2: [[PRED_STORE_IF11]]:
+; VEC4_INTERL2-NEXT: [[TMP22:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP22]], i64 20
+; VEC4_INTERL2-NEXT: [[TMP24:%.*]] = fadd fast float [[DOTCAST1]], 5.000000e+00
+; VEC4_INTERL2-NEXT: store float [[TMP24]], ptr [[TMP23]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE12]]:
+; VEC4_INTERL2-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
+; VEC4_INTERL2-NEXT: br i1 [[TMP25]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; VEC4_INTERL2: [[PRED_STORE_IF13]]:
+; VEC4_INTERL2-NEXT: [[TMP26:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP26]], i64 24
+; VEC4_INTERL2-NEXT: [[TMP28:%.*]] = fadd fast float [[DOTCAST1]], 6.000000e+00
+; VEC4_INTERL2-NEXT: store float [[TMP28]], ptr [[TMP27]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE14]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE14]]:
+; VEC4_INTERL2-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
+; VEC4_INTERL2-NEXT: br i1 [[TMP29]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16]]
+; VEC4_INTERL2: [[PRED_STORE_IF15]]:
+; VEC4_INTERL2-NEXT: [[TMP30:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[TMP30]], i64 28
+; VEC4_INTERL2-NEXT: [[TMP32:%.*]] = fadd fast float [[DOTCAST1]], 7.000000e+00
+; VEC4_INTERL2-NEXT: store float [[TMP32]], ptr [[TMP31]], align 4
+; VEC4_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; VEC4_INTERL2: [[PRED_STORE_CONTINUE16]]:
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; VEC4_INTERL2-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL2-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL17:%.*]] = phi float [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC4_INTERL2-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL17]], [[SCALAR_PH]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL17:%.*]] = phi float [ [[DOTCAST]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC4_INTERL2-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], %[[FOR_INC]] ], [ [[BC_RESUME_VAL17]], %[[SCALAR_PH]] ]
; VEC4_INTERL2-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[I]]
; VEC4_INTERL2-NEXT: [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
; VEC4_INTERL2-NEXT: [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
-; VEC4_INTERL2-NEXT: br i1 [[VAR2]], label [[IF_PRED:%.*]], label [[FOR_INC]]
-; VEC4_INTERL2: if.pred:
+; VEC4_INTERL2-NEXT: br i1 [[VAR2]], label %[[IF_PRED:.*]], label %[[FOR_INC]]
+; VEC4_INTERL2: [[IF_PRED]]:
; VEC4_INTERL2-NEXT: store float [[J]], ptr [[VAR0]], align 4
-; VEC4_INTERL2-NEXT: br label [[FOR_INC]]
-; VEC4_INTERL2: for.inc:
+; VEC4_INTERL2-NEXT: br label %[[FOR_INC]]
+; VEC4_INTERL2: [[FOR_INC]]:
; VEC4_INTERL2-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; VEC4_INTERL2-NEXT: [[J_NEXT]] = fadd fast float [[J]], 1.000000e+00
; VEC4_INTERL2-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; VEC4_INTERL2-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
-; VEC4_INTERL2: for.end:
+; VEC4_INTERL2-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
+; VEC4_INTERL2: [[FOR_END]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @non_primary_iv_float_scalar(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; VEC1_INTERL2-LABEL: define void @non_primary_iv_float_scalar(
+; VEC1_INTERL2-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC1_INTERL2-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 4
; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP0]], align 4
; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP2]], align 4
; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = fcmp fast oeq float [[TMP3]], 0.000000e+00
; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = fcmp fast oeq float [[TMP4]], 0.000000e+00
-; VEC1_INTERL2-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; VEC1_INTERL2: pred.store.if:
+; VEC1_INTERL2-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC1_INTERL2: [[PRED_STORE_IF]]:
; VEC1_INTERL2-NEXT: store float [[DOTCAST1]], ptr [[TMP0]], align 4
-; VEC1_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE]]
-; VEC1_INTERL2: pred.store.continue:
-; VEC1_INTERL2-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; VEC1_INTERL2: pred.store.if2:
+; VEC1_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC1_INTERL2: [[PRED_STORE_CONTINUE]]:
+; VEC1_INTERL2-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; VEC1_INTERL2: [[PRED_STORE_IF2]]:
; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fadd fast float [[DOTCAST1]], 1.000000e+00
; VEC1_INTERL2-NEXT: store float [[TMP7]], ptr [[TMP2]], align 4
-; VEC1_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE3]]
-; VEC1_INTERL2: pred.store.continue3:
+; VEC1_INTERL2-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; VEC1_INTERL2: [[PRED_STORE_CONTINUE3]]:
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL4:%.*]] = phi float [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC1_INTERL2-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL4:%.*]] = phi float [ [[DOTCAST]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC1_INTERL2-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], %[[FOR_INC]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
; VEC1_INTERL2-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[I]]
; VEC1_INTERL2-NEXT: [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
; VEC1_INTERL2-NEXT: [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
-; VEC1_INTERL2-NEXT: br i1 [[VAR2]], label [[IF_PRED:%.*]], label [[FOR_INC]]
-; VEC1_INTERL2: if.pred:
+; VEC1_INTERL2-NEXT: br i1 [[VAR2]], label %[[IF_PRED:.*]], label %[[FOR_INC]]
+; VEC1_INTERL2: [[IF_PRED]]:
; VEC1_INTERL2-NEXT: store float [[J]], ptr [[VAR0]], align 4
-; VEC1_INTERL2-NEXT: br label [[FOR_INC]]
-; VEC1_INTERL2: for.inc:
+; VEC1_INTERL2-NEXT: br label %[[FOR_INC]]
+; VEC1_INTERL2: [[FOR_INC]]:
; VEC1_INTERL2-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; VEC1_INTERL2-NEXT: [[J_NEXT]] = fadd fast float [[J]], 1.000000e+00
; VEC1_INTERL2-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; VEC1_INTERL2-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
-; VEC1_INTERL2: for.end:
+; VEC1_INTERL2-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
+; VEC1_INTERL2: [[FOR_END]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @non_primary_iv_float_scalar(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @non_primary_iv_float_scalar(
+; VEC2_INTERL1_PRED_STORE-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP0]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = fcmp fast oeq <2 x float> [[WIDE_LOAD]], zeroinitializer
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; VEC2_INTERL1_PRED_STORE: pred.store.if:
-; VEC2_INTERL1_PRED_STORE-NEXT: store float [[DOTCAST2]], ptr [[TMP0]], align 4
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[PRED_STORE_CONTINUE]]
-; VEC2_INTERL1_PRED_STORE: pred.store.continue:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; VEC2_INTERL1_PRED_STORE: pred.store.if2:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 4
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP7:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
-; VEC2_INTERL1_PRED_STORE-NEXT: store float [[TMP7]], ptr [[TMP6]], align 4
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[PRED_STORE_CONTINUE3]]
-; VEC2_INTERL1_PRED_STORE: pred.store.continue3:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC2_INTERL1_PRED_STORE: [[PRED_STORE_IF]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: store float [[DOTCAST1]], ptr [[TMP0]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC2_INTERL1_PRED_STORE: [[PRED_STORE_CONTINUE]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; VEC2_INTERL1_PRED_STORE: [[PRED_STORE_IF2]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i64 4
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = fadd fast float [[DOTCAST1]], 1.000000e+00
+; VEC2_INTERL1_PRED_STORE-NEXT: store float [[TMP6]], ptr [[TMP5]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; VEC2_INTERL1_PRED_STORE: [[PRED_STORE_CONTINUE3]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[J:%.*]] = phi float [ [[J_NEXT:%.*]], %[[FOR_INC]] ], [ [[DOTCAST]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[VAR0:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[I]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
; VEC2_INTERL1_PRED_STORE-NEXT: [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[VAR2]], label [[IF_PRED:%.*]], label [[FOR_INC]]
-; VEC2_INTERL1_PRED_STORE: if.pred:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[VAR2]], label %[[IF_PRED:.*]], label %[[FOR_INC]]
+; VEC2_INTERL1_PRED_STORE: [[IF_PRED]]:
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[J]], ptr [[VAR0]], align 4
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[FOR_INC]]
-; VEC2_INTERL1_PRED_STORE: for.inc:
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[FOR_INC]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_INC]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[J_NEXT]] = fadd fast float [[J]], 1.000000e+00
; VEC2_INTERL1_PRED_STORE-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: for.end:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_END]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -1633,76 +1657,80 @@ for.end:
}
define i32 @float_induction_with_dbg_on_fadd(ptr %dst) {
-; VEC4_INTERL1-LABEL: @float_induction_with_dbg_on_fadd(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: br label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL1-LABEL: define i32 @float_induction_with_dbg_on_fadd(
+; VEC4_INTERL1-SAME: ptr [[DST:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = getelementptr [4 x i8], ptr null, i64 [[INDEX]]
; VEC4_INTERL1-NEXT: store <4 x float> poison, ptr [[TMP0]], align 8
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; VEC4_INTERL1-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; VEC4_INTERL1: middle.block:
-; VEC4_INTERL1-NEXT: br label [[LOOP:%.*]]
-; VEC4_INTERL1: exit:
+; VEC4_INTERL1-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
+; VEC4_INTERL1-NEXT: br label %[[EXIT:.*]]
+; VEC4_INTERL1: [[EXIT]]:
; VEC4_INTERL1-NEXT: ret i32 0
;
-; VEC4_INTERL2-LABEL: @float_induction_with_dbg_on_fadd(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: br label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL2-LABEL: define i32 @float_induction_with_dbg_on_fadd(
+; VEC4_INTERL2-SAME: ptr [[DST:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = getelementptr [4 x i8], ptr null, i64 [[INDEX]]
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 16
; VEC4_INTERL2-NEXT: store <4 x float> poison, ptr [[TMP0]], align 8
; VEC4_INTERL2-NEXT: store <4 x float> zeroinitializer, ptr [[TMP1]], align 8
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; VEC4_INTERL2-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; VEC4_INTERL2: middle.block:
-; VEC4_INTERL2-NEXT: br label [[LOOP:%.*]]
-; VEC4_INTERL2: exit:
+; VEC4_INTERL2-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
+; VEC4_INTERL2-NEXT: br label %[[EXIT:.*]]
+; VEC4_INTERL2: [[EXIT]]:
; VEC4_INTERL2-NEXT: ret i32 0
;
-; VEC1_INTERL2-LABEL: @float_induction_with_dbg_on_fadd(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: br label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC1_INTERL2-LABEL: define i32 @float_induction_with_dbg_on_fadd(
+; VEC1_INTERL2-SAME: ptr [[DST:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*:]]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = sitofp i64 [[INDEX]] to float
; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = call reassoc float @llvm.copysign.f32(float 0.000000e+00, float [[DOTCAST]])
; VEC1_INTERL2-NEXT: [[OFFSET_IDX:%.*]] = fadd reassoc float [[TMP0]], 0.000000e+00
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = getelementptr [4 x i8], ptr null, i64 [[INDEX]]
; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = getelementptr [4 x i8], ptr null, i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 4
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i64 4
; VEC1_INTERL2-NEXT: store float poison, ptr [[TMP1]], align 8
-; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP4]], align 8
+; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP3]], align 8
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; VEC1_INTERL2-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; VEC1_INTERL2: middle.block:
-; VEC1_INTERL2-NEXT: br label [[LOOP:%.*]]
-; VEC1_INTERL2: exit:
+; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
+; VEC1_INTERL2-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
+; VEC1_INTERL2-NEXT: br label %[[EXIT:.*]]
+; VEC1_INTERL2: [[EXIT]]:
; VEC1_INTERL2-NEXT: ret i32 0
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @float_induction_with_dbg_on_fadd(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-LABEL: define i32 @float_induction_with_dbg_on_fadd(
+; VEC2_INTERL1_PRED_STORE-SAME: ptr [[DST:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = getelementptr [4 x i8], ptr null, i64 [[INDEX]]
; VEC2_INTERL1_PRED_STORE-NEXT: store <2 x float> poison, ptr [[TMP0]], align 8
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP1]], label [[EXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: exit:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP1]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[EXIT]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret i32 0
;
entry:
@@ -1723,18 +1751,19 @@ exit:
}
define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float %fpinc, i32 %N) {
-; VEC4_INTERL1-LABEL: @fp_iv_used_in_gep_fadd(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC4_INTERL1-LABEL: define void @fp_iv_used_in_gep_fadd(
+; VEC4_INTERL1-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC:%.*]], [[DOTCAST]]
-; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT]], [[TMP3]]
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
@@ -1744,10 +1773,10 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = fmul fast float [[FPINC]], 4.000000e+00
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x float> poison, float [[TMP6]], i64 0
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT3]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[DOTCAST5:%.*]] = sitofp i64 [[INDEX]] to float
; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = fmul fast float [[FPINC]], [[DOTCAST5]]
; VEC4_INTERL1-NEXT: [[OFFSET_IDX:%.*]] = fadd fast float [[INIT]], [[TMP7]]
@@ -1762,7 +1791,7 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[TMP13]], i64 2
; VEC4_INTERL1-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[TMP13]], i64 3
; VEC4_INTERL1-NEXT: [[TMP18:%.*]] = sext i32 [[TMP14]] to i64
-; VEC4_INTERL1-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP18]]
+; VEC4_INTERL1-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP18]]
; VEC4_INTERL1-NEXT: [[TMP20:%.*]] = sext i32 [[TMP15]] to i64
; VEC4_INTERL1-NEXT: [[TMP21:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP20]]
; VEC4_INTERL1-NEXT: [[TMP22:%.*]] = sext i32 [[TMP16]] to i64
@@ -1776,17 +1805,17 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; VEC4_INTERL1-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL7:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL7]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL7:%.*]] = phi float [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL7]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC4_INTERL1-NEXT: [[TMP27:%.*]] = sext i32 [[C]] to i64
; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP27]]
@@ -1795,33 +1824,34 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; VEC4_INTERL1: exit:
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; VEC4_INTERL1: [[EXIT]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_used_in_gep_fadd(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC4_INTERL2-LABEL: define void @fp_iv_used_in_gep_fadd(
+; VEC4_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584
-; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[FPINC:%.*]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
-; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT]], [[TMP3]]
; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT2]], [[TMP6]]
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fadd fast <4 x float> [[VEC_IND]], [[TMP5]]
; VEC4_INTERL2-NEXT: [[DOTCAST3:%.*]] = sitofp i64 [[INDEX]] to float
; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast float [[FPINC]], [[DOTCAST3]]
@@ -1850,7 +1880,7 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL2-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP26]], i64 2
; VEC4_INTERL2-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP26]], i64 3
; VEC4_INTERL2-NEXT: [[TMP31:%.*]] = sext i32 [[TMP22]] to i64
-; VEC4_INTERL2-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP31]]
+; VEC4_INTERL2-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP31]]
; VEC4_INTERL2-NEXT: [[TMP33:%.*]] = sext i32 [[TMP23]] to i64
; VEC4_INTERL2-NEXT: [[TMP34:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP33]]
; VEC4_INTERL2-NEXT: [[TMP35:%.*]] = sext i32 [[TMP24]] to i64
@@ -1876,17 +1906,17 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fadd fast <4 x float> [[STEP_ADD]], [[TMP5]]
; VEC4_INTERL2-NEXT: [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP47]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: br i1 [[TMP47]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL5]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC4_INTERL2-NEXT: [[TMP48:%.*]] = sext i32 [[C]] to i64
; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP48]]
@@ -1895,25 +1925,26 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; VEC4_INTERL2: exit:
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; VEC4_INTERL2: [[EXIT]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_used_in_gep_fadd(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC1_INTERL2-LABEL: define void @fp_iv_used_in_gep_fadd(
+; VEC1_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC:%.*]], [[DOTCAST]]
-; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
+; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT]], [[TMP3]]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]]
; VEC1_INTERL2-NEXT: [[OFFSET_IDX:%.*]] = fadd fast float [[INIT]], [[TMP5]]
@@ -1921,24 +1952,24 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fptoui float [[OFFSET_IDX]] to i32
; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = fptoui float [[TMP6]] to i32
; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = sext i32 [[TMP7]] to i64
-; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP9]]
+; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP9]]
; VEC1_INTERL2-NEXT: [[TMP11:%.*]] = sext i32 [[TMP8]] to i64
; VEC1_INTERL2-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP11]]
; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP10]], align 4
; VEC1_INTERL2-NEXT: store float [[TMP6]], ptr [[TMP12]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC1_INTERL2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC1_INTERL2-NEXT: [[TMP14:%.*]] = sext i32 [[C]] to i64
; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP14]]
@@ -1947,22 +1978,23 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; VEC1_INTERL2: exit:
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; VEC1_INTERL2: [[EXIT]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_used_in_gep_fadd(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_used_in_gep_fadd(
+; VEC2_INTERL1_PRED_STORE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC:%.*]], [[DOTCAST]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT:%.*]], [[TMP3]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = fadd fast float [[INIT]], [[TMP3]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
@@ -1972,10 +2004,10 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP6]], i64 0
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST5:%.*]] = sitofp i64 [[INDEX]] to float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP7:%.*]] = fmul fast float [[FPINC]], [[DOTCAST5]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[OFFSET_IDX:%.*]] = fadd fast float [[INIT]], [[TMP7]]
@@ -1984,7 +2016,7 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP9]], i64 0
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP9]], i64 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP12:%.*]] = sext i32 [[TMP10]] to i64
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP12]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP12]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP14:%.*]] = sext i32 [[TMP11]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP14]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[OFFSET_IDX]], ptr [[TMP13]], align 4
@@ -1992,13 +2024,13 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fadd fast <2 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP17:%.*]] = sext i32 [[C]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP17]]
@@ -2007,8 +2039,8 @@ define void @fp_iv_used_in_gep_fadd(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: exit:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[EXIT]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -2031,18 +2063,19 @@ exit:
}
define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float %fpinc, i32 %N) {
-; VEC4_INTERL1-LABEL: @fp_iv_used_in_gep_fsub(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC4_INTERL1-LABEL: define void @fp_iv_used_in_gep_fsub(
+; VEC4_INTERL1-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC4_INTERL1-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC4_INTERL1-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3
-; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL1: vector.ph:
+; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL1: [[VECTOR_PH]]:
; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588
; VEC4_INTERL1-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC:%.*]], [[DOTCAST]]
-; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT:%.*]], [[TMP3]]
+; VEC4_INTERL1-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
+; VEC4_INTERL1-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT]], [[TMP3]]
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
@@ -2052,10 +2085,10 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = fmul fast float [[FPINC]], 4.000000e+00
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x float> poison, float [[TMP6]], i64 0
; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT3]], <4 x float> poison, <4 x i32> zeroinitializer
-; VEC4_INTERL1-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL1: vector.body:
-; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL1: [[VECTOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[DOTCAST5:%.*]] = sitofp i64 [[INDEX]] to float
; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = fmul fast float [[FPINC]], [[DOTCAST5]]
; VEC4_INTERL1-NEXT: [[OFFSET_IDX:%.*]] = fsub fast float [[INIT]], [[TMP7]]
@@ -2070,7 +2103,7 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[TMP13]], i64 2
; VEC4_INTERL1-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[TMP13]], i64 3
; VEC4_INTERL1-NEXT: [[TMP18:%.*]] = sext i32 [[TMP14]] to i64
-; VEC4_INTERL1-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP18]]
+; VEC4_INTERL1-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP18]]
; VEC4_INTERL1-NEXT: [[TMP20:%.*]] = sext i32 [[TMP15]] to i64
; VEC4_INTERL1-NEXT: [[TMP21:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP20]]
; VEC4_INTERL1-NEXT: [[TMP22:%.*]] = sext i32 [[TMP16]] to i64
@@ -2084,17 +2117,17 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VEC4_INTERL1-NEXT: [[VEC_IND_NEXT]] = fsub fast <4 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; VEC4_INTERL1-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; VEC4_INTERL1: middle.block:
+; VEC4_INTERL1-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VEC4_INTERL1: [[MIDDLE_BLOCK]]:
; VEC4_INTERL1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL1: scalar.ph:
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL7:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL7]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL1: [[SCALAR_PH]]:
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC4_INTERL1-NEXT: [[BC_RESUME_VAL7:%.*]] = phi float [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL7]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC4_INTERL1-NEXT: [[TMP27:%.*]] = sext i32 [[C]] to i64
; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP27]]
@@ -2103,33 +2136,34 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; VEC4_INTERL1: exit:
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; VEC4_INTERL1: [[EXIT]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_used_in_gep_fsub(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC4_INTERL2-LABEL: define void @fp_iv_used_in_gep_fsub(
+; VEC4_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC4_INTERL2-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC4_INTERL2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7
-; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC4_INTERL2: vector.ph:
+; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC4_INTERL2: [[VECTOR_PH]]:
; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584
-; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[FPINC:%.*]], i64 0
+; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[FPINC]], i64 0
; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
; VEC4_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
-; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT:%.*]], [[TMP3]]
+; VEC4_INTERL2-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT]], [[TMP3]]
; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], splat (float 4.000000e+00)
; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[INIT]], i64 0
; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
; VEC4_INTERL2-NEXT: [[INDUCTION:%.*]] = fsub fast <4 x float> [[BROADCAST_SPLAT2]], [[TMP6]]
-; VEC4_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC4_INTERL2: vector.body:
-; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC4_INTERL2: [[VECTOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[STEP_ADD:%.*]] = fsub fast <4 x float> [[VEC_IND]], [[TMP5]]
; VEC4_INTERL2-NEXT: [[DOTCAST3:%.*]] = sitofp i64 [[INDEX]] to float
; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast float [[FPINC]], [[DOTCAST3]]
@@ -2145,82 +2179,83 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC4_INTERL2-NEXT: [[TMP16:%.*]] = fsub fast float [[OFFSET_IDX]], [[TMP15]]
; VEC4_INTERL2-NEXT: [[TMP17:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
; VEC4_INTERL2-NEXT: [[TMP18:%.*]] = fsub fast float [[OFFSET_IDX]], [[TMP17]]
-; VEC4_INTERL2-NEXT: [[TMP20:%.*]] = fsub fast float [[OFFSET_IDX]], [[FPINC]]
-; VEC4_INTERL2-NEXT: [[TMP21:%.*]] = fptoui <4 x float> [[VEC_IND]] to <4 x i32>
-; VEC4_INTERL2-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[TMP21]], i64 0
-; VEC4_INTERL2-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[TMP21]], i64 1
-; VEC4_INTERL2-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP21]], i64 2
-; VEC4_INTERL2-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP21]], i64 3
-; VEC4_INTERL2-NEXT: [[TMP26:%.*]] = fptoui <4 x float> [[STEP_ADD]] to <4 x i32>
-; VEC4_INTERL2-NEXT: [[TMP27:%.*]] = extractelement <4 x i32> [[TMP26]], i64 0
-; VEC4_INTERL2-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP26]], i64 1
-; VEC4_INTERL2-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP26]], i64 2
-; VEC4_INTERL2-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP26]], i64 3
-; VEC4_INTERL2-NEXT: [[TMP31:%.*]] = sext i32 [[TMP22]] to i64
-; VEC4_INTERL2-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP31]]
-; VEC4_INTERL2-NEXT: [[TMP33:%.*]] = sext i32 [[TMP23]] to i64
-; VEC4_INTERL2-NEXT: [[TMP34:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP33]]
-; VEC4_INTERL2-NEXT: [[TMP35:%.*]] = sext i32 [[TMP24]] to i64
-; VEC4_INTERL2-NEXT: [[TMP36:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP35]]
-; VEC4_INTERL2-NEXT: [[TMP37:%.*]] = sext i32 [[TMP25]] to i64
-; VEC4_INTERL2-NEXT: [[TMP38:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP37]]
-; VEC4_INTERL2-NEXT: [[TMP39:%.*]] = sext i32 [[TMP27]] to i64
-; VEC4_INTERL2-NEXT: [[TMP40:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP39]]
-; VEC4_INTERL2-NEXT: [[TMP41:%.*]] = sext i32 [[TMP28]] to i64
-; VEC4_INTERL2-NEXT: [[TMP42:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP41]]
-; VEC4_INTERL2-NEXT: [[TMP43:%.*]] = sext i32 [[TMP29]] to i64
-; VEC4_INTERL2-NEXT: [[TMP44:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP43]]
-; VEC4_INTERL2-NEXT: [[TMP45:%.*]] = sext i32 [[TMP30]] to i64
-; VEC4_INTERL2-NEXT: [[TMP46:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP45]]
-; VEC4_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP32]], align 4
-; VEC4_INTERL2-NEXT: store float [[TMP8]], ptr [[TMP34]], align 4
-; VEC4_INTERL2-NEXT: store float [[TMP10]], ptr [[TMP36]], align 4
-; VEC4_INTERL2-NEXT: store float [[TMP12]], ptr [[TMP38]], align 4
-; VEC4_INTERL2-NEXT: store float [[TMP14]], ptr [[TMP40]], align 4
-; VEC4_INTERL2-NEXT: store float [[TMP16]], ptr [[TMP42]], align 4
-; VEC4_INTERL2-NEXT: store float [[TMP18]], ptr [[TMP44]], align 4
-; VEC4_INTERL2-NEXT: store float [[TMP20]], ptr [[TMP46]], align 4
+; VEC4_INTERL2-NEXT: [[TMP19:%.*]] = fsub fast float [[OFFSET_IDX]], [[FPINC]]
+; VEC4_INTERL2-NEXT: [[TMP20:%.*]] = fptoui <4 x float> [[VEC_IND]] to <4 x i32>
+; VEC4_INTERL2-NEXT: [[TMP21:%.*]] = extractelement <4 x i32> [[TMP20]], i64 0
+; VEC4_INTERL2-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[TMP20]], i64 1
+; VEC4_INTERL2-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[TMP20]], i64 2
+; VEC4_INTERL2-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP20]], i64 3
+; VEC4_INTERL2-NEXT: [[TMP25:%.*]] = fptoui <4 x float> [[STEP_ADD]] to <4 x i32>
+; VEC4_INTERL2-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP25]], i64 0
+; VEC4_INTERL2-NEXT: [[TMP27:%.*]] = extractelement <4 x i32> [[TMP25]], i64 1
+; VEC4_INTERL2-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP25]], i64 2
+; VEC4_INTERL2-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP25]], i64 3
+; VEC4_INTERL2-NEXT: [[TMP30:%.*]] = sext i32 [[TMP21]] to i64
+; VEC4_INTERL2-NEXT: [[TMP31:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP30]]
+; VEC4_INTERL2-NEXT: [[TMP32:%.*]] = sext i32 [[TMP22]] to i64
+; VEC4_INTERL2-NEXT: [[TMP33:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP32]]
+; VEC4_INTERL2-NEXT: [[TMP34:%.*]] = sext i32 [[TMP23]] to i64
+; VEC4_INTERL2-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP34]]
+; VEC4_INTERL2-NEXT: [[TMP36:%.*]] = sext i32 [[TMP24]] to i64
+; VEC4_INTERL2-NEXT: [[TMP37:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP36]]
+; VEC4_INTERL2-NEXT: [[TMP38:%.*]] = sext i32 [[TMP26]] to i64
+; VEC4_INTERL2-NEXT: [[TMP39:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP38]]
+; VEC4_INTERL2-NEXT: [[TMP40:%.*]] = sext i32 [[TMP27]] to i64
+; VEC4_INTERL2-NEXT: [[TMP41:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP40]]
+; VEC4_INTERL2-NEXT: [[TMP42:%.*]] = sext i32 [[TMP28]] to i64
+; VEC4_INTERL2-NEXT: [[TMP43:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP42]]
+; VEC4_INTERL2-NEXT: [[TMP44:%.*]] = sext i32 [[TMP29]] to i64
+; VEC4_INTERL2-NEXT: [[TMP45:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP44]]
+; VEC4_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP31]], align 4
+; VEC4_INTERL2-NEXT: store float [[TMP8]], ptr [[TMP33]], align 4
+; VEC4_INTERL2-NEXT: store float [[TMP10]], ptr [[TMP35]], align 4
+; VEC4_INTERL2-NEXT: store float [[TMP12]], ptr [[TMP37]], align 4
+; VEC4_INTERL2-NEXT: store float [[TMP14]], ptr [[TMP39]], align 4
+; VEC4_INTERL2-NEXT: store float [[TMP16]], ptr [[TMP41]], align 4
+; VEC4_INTERL2-NEXT: store float [[TMP18]], ptr [[TMP43]], align 4
+; VEC4_INTERL2-NEXT: store float [[TMP19]], ptr [[TMP45]], align 4
; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; VEC4_INTERL2-NEXT: [[VEC_IND_NEXT]] = fsub fast <4 x float> [[STEP_ADD]], [[TMP5]]
-; VEC4_INTERL2-NEXT: [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[TMP47]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; VEC4_INTERL2: middle.block:
+; VEC4_INTERL2-NEXT: [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC4_INTERL2-NEXT: br i1 [[TMP46]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VEC4_INTERL2: [[MIDDLE_BLOCK]]:
; VEC4_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; VEC4_INTERL2: scalar.ph:
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL5]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VEC4_INTERL2: [[SCALAR_PH]]:
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC4_INTERL2-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
-; VEC4_INTERL2-NEXT: [[TMP48:%.*]] = sext i32 [[C]] to i64
-; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP48]]
+; VEC4_INTERL2-NEXT: [[TMP47:%.*]] = sext i32 [[C]] to i64
+; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP47]]
; VEC4_INTERL2-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
; VEC4_INTERL2-NEXT: [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; VEC4_INTERL2: exit:
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; VEC4_INTERL2: [[EXIT]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_used_in_gep_fsub(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC1_INTERL2-LABEL: define void @fp_iv_used_in_gep_fsub(
+; VEC1_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC1_INTERL2-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC1_INTERL2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0
-; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC1_INTERL2: vector.ph:
+; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC1_INTERL2: [[VECTOR_PH]]:
; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590
; VEC1_INTERL2-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC:%.*]], [[DOTCAST]]
-; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT:%.*]], [[TMP3]]
-; VEC1_INTERL2-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC1_INTERL2: vector.body:
-; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
+; VEC1_INTERL2-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT]], [[TMP3]]
+; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC1_INTERL2: [[VECTOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
; VEC1_INTERL2-NEXT: [[TMP5:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]]
; VEC1_INTERL2-NEXT: [[OFFSET_IDX:%.*]] = fsub fast float [[INIT]], [[TMP5]]
@@ -2228,24 +2263,24 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fptoui float [[OFFSET_IDX]] to i32
; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = fptoui float [[TMP6]] to i32
; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = sext i32 [[TMP7]] to i64
-; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP9]]
+; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP9]]
; VEC1_INTERL2-NEXT: [[TMP11:%.*]] = sext i32 [[TMP8]] to i64
; VEC1_INTERL2-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP11]]
; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX]], ptr [[TMP10]], align 4
; VEC1_INTERL2-NEXT: store float [[TMP6]], ptr [[TMP12]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC1_INTERL2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; VEC1_INTERL2: middle.block:
+; VEC1_INTERL2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VEC1_INTERL2: [[MIDDLE_BLOCK]]:
; VEC1_INTERL2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; VEC1_INTERL2: scalar.ph:
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VEC1_INTERL2: [[SCALAR_PH]]:
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC1_INTERL2-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC1_INTERL2-NEXT: [[TMP14:%.*]] = sext i32 [[C]] to i64
; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP14]]
@@ -2254,22 +2289,23 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; VEC1_INTERL2: exit:
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; VEC1_INTERL2: [[EXIT]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_used_in_gep_fsub(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_used_in_gep_fsub(
+; VEC2_INTERL1_PRED_STORE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.ph:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_PH]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC:%.*]], [[DOTCAST]]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT:%.*]], [[TMP3]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP4:%.*]] = fsub fast float [[INIT]], [[TMP3]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
@@ -2279,10 +2315,10 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP6:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP6]], i64 0
; VEC2_INTERL1_PRED_STORE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[VECTOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[DOTCAST5:%.*]] = sitofp i64 [[INDEX]] to float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP7:%.*]] = fmul fast float [[FPINC]], [[DOTCAST5]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[OFFSET_IDX:%.*]] = fsub fast float [[INIT]], [[TMP7]]
@@ -2291,7 +2327,7 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP9]], i64 0
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP9]], i64 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP12:%.*]] = sext i32 [[TMP10]] to i64
-; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP12]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP12]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP14:%.*]] = sext i32 [[TMP11]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP14]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[OFFSET_IDX]], ptr [[TMP13]], align 4
@@ -2299,13 +2335,13 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC2_INTERL1_PRED_STORE-NEXT: [[VEC_IND_NEXT]] = fsub fast <2 x float> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: middle.block:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[MIDDLE_BLOCK]]:
; VEC2_INTERL1_PRED_STORE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[ENTRY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP17:%.*]] = sext i32 [[C]] to i64
; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP17]]
@@ -2314,8 +2350,8 @@ define void @fp_iv_used_in_gep_fsub(float %init, ptr noalias nocapture %A, float
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; VEC2_INTERL1_PRED_STORE: exit:
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; VEC2_INTERL1_PRED_STORE: [[EXIT]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
@@ -2338,76 +2374,80 @@ exit:
}
define void @fp_iv_used_in_gep_fmul(float %init, ptr noalias nocapture %A, float %fpinc, i32 %N) {
-; VEC4_INTERL1-LABEL: @fp_iv_used_in_gep_fmul(
-; VEC4_INTERL1-NEXT: entry:
-; VEC4_INTERL1-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL1: for.body:
-; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[INIT:%.*]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL1-LABEL: define void @fp_iv_used_in_gep_fmul(
+; VEC4_INTERL1-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL1-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL1-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL1: [[FOR_BODY]]:
+; VEC4_INTERL1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL1-NEXT: [[X_05:%.*]] = phi float [ [[INIT]], %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL1-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC4_INTERL1-NEXT: [[TMP0:%.*]] = sext i32 [[C]] to i64
-; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP0]]
+; VEC4_INTERL1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP0]]
; VEC4_INTERL1-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
-; VEC4_INTERL1-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC:%.*]]
+; VEC4_INTERL1-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC]]
; VEC4_INTERL1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N:%.*]], [[LFTR_WIDEIV]]
-; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR_BODY]]
-; VEC4_INTERL1: exit:
+; VEC4_INTERL1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
+; VEC4_INTERL1-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; VEC4_INTERL1: [[EXIT]]:
; VEC4_INTERL1-NEXT: ret void
;
-; VEC4_INTERL2-LABEL: @fp_iv_used_in_gep_fmul(
-; VEC4_INTERL2-NEXT: entry:
-; VEC4_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC4_INTERL2: for.body:
-; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[INIT:%.*]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC4_INTERL2-LABEL: define void @fp_iv_used_in_gep_fmul(
+; VEC4_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC4_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC4_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC4_INTERL2: [[FOR_BODY]]:
+; VEC4_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC4_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[INIT]], %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC4_INTERL2-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC4_INTERL2-NEXT: [[TMP0:%.*]] = sext i32 [[C]] to i64
-; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP0]]
+; VEC4_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP0]]
; VEC4_INTERL2-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
-; VEC4_INTERL2-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC:%.*]]
+; VEC4_INTERL2-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC]]
; VEC4_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC4_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N:%.*]], [[LFTR_WIDEIV]]
-; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR_BODY]]
-; VEC4_INTERL2: exit:
+; VEC4_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
+; VEC4_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; VEC4_INTERL2: [[EXIT]]:
; VEC4_INTERL2-NEXT: ret void
;
-; VEC1_INTERL2-LABEL: @fp_iv_used_in_gep_fmul(
-; VEC1_INTERL2-NEXT: entry:
-; VEC1_INTERL2-NEXT: br label [[FOR_BODY:%.*]]
-; VEC1_INTERL2: for.body:
-; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[INIT:%.*]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-LABEL: define void @fp_iv_used_in_gep_fmul(
+; VEC1_INTERL2-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC1_INTERL2-NEXT: [[ENTRY:.*]]:
+; VEC1_INTERL2-NEXT: br label %[[FOR_BODY:.*]]
+; VEC1_INTERL2: [[FOR_BODY]]:
+; VEC1_INTERL2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT: [[X_05:%.*]] = phi float [ [[INIT]], %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC1_INTERL2-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC1_INTERL2-NEXT: [[TMP0:%.*]] = sext i32 [[C]] to i64
-; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP0]]
+; VEC1_INTERL2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP0]]
; VEC1_INTERL2-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
-; VEC1_INTERL2-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC:%.*]]
+; VEC1_INTERL2-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC]]
; VEC1_INTERL2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC1_INTERL2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N:%.*]], [[LFTR_WIDEIV]]
-; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR_BODY]]
-; VEC1_INTERL2: exit:
+; VEC1_INTERL2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
+; VEC1_INTERL2-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; VEC1_INTERL2: [[EXIT]]:
; VEC1_INTERL2-NEXT: ret void
;
-; VEC2_INTERL1_PRED_STORE-LABEL: @fp_iv_used_in_gep_fmul(
-; VEC2_INTERL1_PRED_STORE-NEXT: entry:
-; VEC2_INTERL1_PRED_STORE-NEXT: br label [[FOR_BODY:%.*]]
-; VEC2_INTERL1_PRED_STORE: for.body:
-; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[INIT:%.*]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-LABEL: define void @fp_iv_used_in_gep_fmul(
+; VEC2_INTERL1_PRED_STORE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], float [[FPINC:%.*]], i32 [[N:%.*]]) {
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ENTRY:.*]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: br label %[[FOR_BODY:.*]]
+; VEC2_INTERL1_PRED_STORE: [[FOR_BODY]]:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[X_05:%.*]] = phi float [ [[INIT]], %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
; VEC2_INTERL1_PRED_STORE-NEXT: [[C:%.*]] = fptoui float [[X_05]] to i32
; VEC2_INTERL1_PRED_STORE-NEXT: [[TMP0:%.*]] = sext i32 [[C]] to i64
-; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP0]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP0]]
; VEC2_INTERL1_PRED_STORE-NEXT: store float [[X_05]], ptr [[ARRAYIDX]], align 4
-; VEC2_INTERL1_PRED_STORE-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC:%.*]]
+; VEC2_INTERL1_PRED_STORE-NEXT: [[ADD]] = fmul fast float [[X_05]], [[FPINC]]
; VEC2_INTERL1_PRED_STORE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; VEC2_INTERL1_PRED_STORE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N:%.*]], [[LFTR_WIDEIV]]
-; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR_BODY]]
-; VEC2_INTERL1_PRED_STORE: exit:
+; VEC2_INTERL1_PRED_STORE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]
+; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; VEC2_INTERL1_PRED_STORE: [[EXIT]]:
; VEC2_INTERL1_PRED_STORE-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
index bb5273976a7b6..f91bae6b9c2e8 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S -force-vector-width=2 -force-vector-interleave=1 -passes=loop-vectorize,simplifycfg -verify-loop-info -simplifycfg-require-and-preserve-domtree=1 -force-widen-divrem-via-safe-divisor=0 < %s | FileCheck %s
; RUN: opt -S -force-vector-width=1 -force-vector-interleave=2 -passes=loop-vectorize -verify-loop-info -force-widen-divrem-via-safe-divisor=0 < %s | FileCheck %s --check-prefix=UNROLL-NO-VF
@@ -8,12 +8,13 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; instructions permit vectorization and (ii) the creation of an insertelement
; and a Phi node. We check the full 2-element sequence for all predicate instructions.
define void @test(ptr nocapture %asd, ptr nocapture %aud,
-; CHECK-LABEL: @test(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD:%.*]], i64 512
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[AUD:%.*]], i64 512
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[ASR:%.*]], i64 512
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[AUR:%.*]], i64 512
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr captures(none) [[ASD:%.*]], ptr captures(none) [[AUD:%.*]], ptr captures(none) [[ASR:%.*]], ptr captures(none) [[AUR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD]], i64 512
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[AUD]], i64 512
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[ASR]], i64 512
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[AUR]], i64 512
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ASD]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[AUD]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -37,87 +38,87 @@ define void @test(ptr nocapture %asd, ptr nocapture %aud,
; CHECK-NEXT: [[BOUND120:%.*]] = icmp ult ptr [[AUR]], [[SCEVGEP2]]
; CHECK-NEXT: [[FOUND_CONFLICT21:%.*]] = and i1 [[BOUND019]], [[BOUND120]]
; CHECK-NEXT: [[CONFLICT_RDX22:%.*]] = or i1 [[CONFLICT_RDX18]], [[FOUND_CONFLICT21]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX22]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE27:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[AUR]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD24:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META14:![0-9]+]], !noalias [[META15:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD25:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META15]]
-; CHECK-NEXT: [[TMP9:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 23)
-; CHECK-NEXT: [[TMP10:%.*]] = add nsw <2 x i32> [[WIDE_LOAD23]], splat (i32 24)
-; CHECK-NEXT: [[TMP11:%.*]] = add nsw <2 x i32> [[WIDE_LOAD24]], splat (i32 25)
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw <2 x i32> [[WIDE_LOAD25]], splat (i32 26)
-; CHECK-NEXT: [[TMP13:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
-; CHECK: pred.urem.if:
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP9]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = sdiv i32 [[TMP15]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i32> [[WIDE_LOAD23]], i64 0
-; CHECK-NEXT: [[TMP21:%.*]] = udiv i32 [[TMP19]], [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i32> [[TMP11]], i64 0
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i32> [[WIDE_LOAD24]], i64 0
-; CHECK-NEXT: [[TMP25:%.*]] = srem i32 [[TMP23]], [[TMP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> poison, i32 [[TMP25]], i64 0
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <2 x i32> [[TMP12]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <2 x i32> [[WIDE_LOAD25]], i64 0
-; CHECK-NEXT: [[TMP29:%.*]] = urem i32 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> poison, i32 [[TMP29]], i64 0
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE]]
-; CHECK: pred.urem.continue:
-; CHECK-NEXT: [[TMP31:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP18]], [[PRED_UREM_IF]] ]
-; CHECK-NEXT: [[TMP32:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP22]], [[PRED_UREM_IF]] ]
-; CHECK-NEXT: [[TMP33:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP26]], [[PRED_UREM_IF]] ]
-; CHECK-NEXT: [[TMP34:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP30]], [[PRED_UREM_IF]] ]
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP13]], i64 1
-; CHECK-NEXT: br i1 [[TMP35]], label [[PRED_UREM_IF26:%.*]], label [[PRED_UREM_CONTINUE27]]
-; CHECK: pred.urem.if26:
-; CHECK-NEXT: [[TMP36:%.*]] = extractelement <2 x i32> [[TMP9]], i64 1
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP38:%.*]] = sdiv i32 [[TMP36]], [[TMP37]]
-; CHECK-NEXT: [[TMP39:%.*]] = insertelement <2 x i32> [[TMP31]], i32 [[TMP38]], i64 1
-; CHECK-NEXT: [[TMP40:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
-; CHECK-NEXT: [[TMP41:%.*]] = extractelement <2 x i32> [[WIDE_LOAD23]], i64 1
-; CHECK-NEXT: [[TMP42:%.*]] = udiv i32 [[TMP40]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = insertelement <2 x i32> [[TMP32]], i32 [[TMP42]], i64 1
-; CHECK-NEXT: [[TMP44:%.*]] = extractelement <2 x i32> [[TMP11]], i64 1
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <2 x i32> [[WIDE_LOAD24]], i64 1
-; CHECK-NEXT: [[TMP46:%.*]] = srem i32 [[TMP44]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = insertelement <2 x i32> [[TMP33]], i32 [[TMP46]], i64 1
-; CHECK-NEXT: [[TMP48:%.*]] = extractelement <2 x i32> [[TMP12]], i64 1
-; CHECK-NEXT: [[TMP49:%.*]] = extractelement <2 x i32> [[WIDE_LOAD25]], i64 1
-; CHECK-NEXT: [[TMP50:%.*]] = urem i32 [[TMP48]], [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = insertelement <2 x i32> [[TMP34]], i32 [[TMP50]], i64 1
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE27]]
-; CHECK: pred.urem.continue27:
-; CHECK-NEXT: [[TMP52:%.*]] = phi <2 x i32> [ [[TMP31]], [[PRED_UREM_CONTINUE]] ], [ [[TMP39]], [[PRED_UREM_IF26]] ]
-; CHECK-NEXT: [[TMP53:%.*]] = phi <2 x i32> [ [[TMP32]], [[PRED_UREM_CONTINUE]] ], [ [[TMP43]], [[PRED_UREM_IF26]] ]
-; CHECK-NEXT: [[TMP54:%.*]] = phi <2 x i32> [ [[TMP33]], [[PRED_UREM_CONTINUE]] ], [ [[TMP47]], [[PRED_UREM_IF26]] ]
-; CHECK-NEXT: [[TMP55:%.*]] = phi <2 x i32> [ [[TMP34]], [[PRED_UREM_CONTINUE]] ], [ [[TMP51]], [[PRED_UREM_IF26]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP13]], <2 x i32> [[TMP52]], <2 x i32> [[TMP9]]
-; CHECK-NEXT: [[PREDPHI28:%.*]] = select <2 x i1> [[TMP13]], <2 x i32> [[TMP53]], <2 x i32> [[TMP10]]
-; CHECK-NEXT: [[PREDPHI29:%.*]] = select <2 x i1> [[TMP13]], <2 x i32> [[TMP54]], <2 x i32> [[TMP11]]
-; CHECK-NEXT: [[PREDPHI30:%.*]] = select <2 x i1> [[TMP13]], <2 x i32> [[TMP55]], <2 x i32> [[TMP12]]
-; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP1]], align 4, !alias.scope [[META5]], !noalias [[META8]]
-; CHECK-NEXT: store <2 x i32> [[PREDPHI28]], ptr [[TMP2]], align 4, !alias.scope [[META12]], !noalias [[META13]]
-; CHECK-NEXT: store <2 x i32> [[PREDPHI29]], ptr [[TMP3]], align 4, !alias.scope [[META14]], !noalias [[META15]]
-; CHECK-NEXT: store <2 x i32> [[PREDPHI30]], ptr [[TMP4]], align 4, !alias.scope [[META15]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX22]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE27:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[AUR]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD24:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META14:![0-9]+]], !noalias [[META15:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD25:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META15]]
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 23)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <2 x i32> [[WIDE_LOAD23]], splat (i32 24)
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw <2 x i32> [[WIDE_LOAD24]], splat (i32 25)
+; CHECK-NEXT: [[TMP7:%.*]] = add nsw <2 x i32> [[WIDE_LOAD25]], splat (i32 26)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE:.*]]
+; CHECK: [[PRED_UREM_IF]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = sdiv i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[WIDE_LOAD23]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = udiv i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP16]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i32> [[WIDE_LOAD24]], i64 0
+; CHECK-NEXT: [[TMP20:%.*]] = srem i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> poison, i32 [[TMP20]], i64 0
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i32> [[WIDE_LOAD25]], i64 0
+; CHECK-NEXT: [[TMP24:%.*]] = urem i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> poison, i32 [[TMP24]], i64 0
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE]]
+; CHECK: [[PRED_UREM_CONTINUE]]:
+; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP13]], %[[PRED_UREM_IF]] ]
+; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP17]], %[[PRED_UREM_IF]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP21]], %[[PRED_UREM_IF]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP25]], %[[PRED_UREM_IF]] ]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP8]], i64 1
+; CHECK-NEXT: br i1 [[TMP30]], label %[[PRED_UREM_IF26:.*]], label %[[PRED_UREM_CONTINUE27]]
+; CHECK: [[PRED_UREM_IF26]]:
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP33:%.*]] = sdiv i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x i32> [[TMP26]], i32 [[TMP33]], i64 1
+; CHECK-NEXT: [[TMP35:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <2 x i32> [[WIDE_LOAD23]], i64 1
+; CHECK-NEXT: [[TMP37:%.*]] = udiv i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TMP37]], i64 1
+; CHECK-NEXT: [[TMP39:%.*]] = extractelement <2 x i32> [[TMP6]], i64 1
+; CHECK-NEXT: [[TMP40:%.*]] = extractelement <2 x i32> [[WIDE_LOAD24]], i64 1
+; CHECK-NEXT: [[TMP41:%.*]] = srem i32 [[TMP39]], [[TMP40]]
+; CHECK-NEXT: [[TMP42:%.*]] = insertelement <2 x i32> [[TMP28]], i32 [[TMP41]], i64 1
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP44:%.*]] = extractelement <2 x i32> [[WIDE_LOAD25]], i64 1
+; CHECK-NEXT: [[TMP45:%.*]] = urem i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT: [[TMP46:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP45]], i64 1
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE27]]
+; CHECK: [[PRED_UREM_CONTINUE27]]:
+; CHECK-NEXT: [[TMP47:%.*]] = phi <2 x i32> [ [[TMP26]], %[[PRED_UREM_CONTINUE]] ], [ [[TMP34]], %[[PRED_UREM_IF26]] ]
+; CHECK-NEXT: [[TMP48:%.*]] = phi <2 x i32> [ [[TMP27]], %[[PRED_UREM_CONTINUE]] ], [ [[TMP38]], %[[PRED_UREM_IF26]] ]
+; CHECK-NEXT: [[TMP49:%.*]] = phi <2 x i32> [ [[TMP28]], %[[PRED_UREM_CONTINUE]] ], [ [[TMP42]], %[[PRED_UREM_IF26]] ]
+; CHECK-NEXT: [[TMP50:%.*]] = phi <2 x i32> [ [[TMP29]], %[[PRED_UREM_CONTINUE]] ], [ [[TMP46]], %[[PRED_UREM_IF26]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP8]], <2 x i32> [[TMP47]], <2 x i32> [[TMP4]]
+; CHECK-NEXT: [[PREDPHI28:%.*]] = select <2 x i1> [[TMP8]], <2 x i32> [[TMP48]], <2 x i32> [[TMP5]]
+; CHECK-NEXT: [[PREDPHI29:%.*]] = select <2 x i1> [[TMP8]], <2 x i32> [[TMP49]], <2 x i32> [[TMP6]]
+; CHECK-NEXT: [[PREDPHI30:%.*]] = select <2 x i1> [[TMP8]], <2 x i32> [[TMP50]], <2 x i32> [[TMP7]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP0]], align 4, !alias.scope [[META5]], !noalias [[META8]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI28]], ptr [[TMP1]], align 4, !alias.scope [[META12]], !noalias [[META13]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI29]], ptr [[TMP2]], align 4, !alias.scope [[META14]], !noalias [[META15]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI30]], ptr [[TMP3]], align 4, !alias.scope [[META15]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP61:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP61]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP51]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
-; CHECK: for.body:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ], [ 0, [[ENTRY]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[ISD:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[IUD:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[ISR:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[INDVARS_IV]]
@@ -131,34 +132,35 @@ define void @test(ptr nocapture %asd, ptr nocapture %aud,
; CHECK-NEXT: [[PSR:%.*]] = add nsw i32 [[LSR]], 25
; CHECK-NEXT: [[PUR:%.*]] = add nsw i32 [[LUR]], 26
; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[LSD]], 100
-; CHECK-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END]]
-; CHECK: if.then:
+; CHECK-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_END]]
+; CHECK: [[IF_THEN]]:
; CHECK-NEXT: [[RSD:%.*]] = sdiv i32 [[PSD]], [[LSD]]
; CHECK-NEXT: [[RUD:%.*]] = udiv i32 [[PUD]], [[LUD]]
; CHECK-NEXT: [[RSR:%.*]] = srem i32 [[PSR]], [[LSR]]
; CHECK-NEXT: [[RUR:%.*]] = urem i32 [[PUR]], [[LUR]]
-; CHECK-NEXT: br label [[IF_END]]
-; CHECK: if.end:
-; CHECK-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], [[IF_THEN]] ], [ [[PSD]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[YUD_0:%.*]] = phi i32 [ [[RUD]], [[IF_THEN]] ], [ [[PUD]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[YSR_0:%.*]] = phi i32 [ [[RSR]], [[IF_THEN]] ], [ [[PSR]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[YUR_0:%.*]] = phi i32 [ [[RUR]], [[IF_THEN]] ], [ [[PUR]], [[FOR_BODY]] ]
+; CHECK-NEXT: br label %[[IF_END]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], %[[IF_THEN]] ], [ [[PSD]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[YUD_0:%.*]] = phi i32 [ [[RUD]], %[[IF_THEN]] ], [ [[PUD]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[YSR_0:%.*]] = phi i32 [ [[RSR]], %[[IF_THEN]] ], [ [[PSR]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[YUR_0:%.*]] = phi i32 [ [[RUR]], %[[IF_THEN]] ], [ [[PUR]], %[[FOR_BODY]] ]
; CHECK-NEXT: store i32 [[YSD_0]], ptr [[ISD]], align 4
; CHECK-NEXT: store i32 [[YUD_0]], ptr [[IUD]], align 4
; CHECK-NEXT: store i32 [[YSR_0]], ptr [[ISR]], align 4
; CHECK-NEXT: store i32 [[YUR_0]], ptr [[IUR]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
;
-; UNROLL-NO-VF-LABEL: @test(
-; UNROLL-NO-VF-NEXT: entry:
-; UNROLL-NO-VF-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
-; UNROLL-NO-VF: vector.memcheck:
-; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD:%.*]], i64 512
-; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[AUD:%.*]], i64 512
-; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[ASR:%.*]], i64 512
-; UNROLL-NO-VF-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[AUR:%.*]], i64 512
+; UNROLL-NO-VF-LABEL: define void @test(
+; UNROLL-NO-VF-SAME: ptr captures(none) [[ASD:%.*]], ptr captures(none) [[AUD:%.*]], ptr captures(none) [[ASR:%.*]], ptr captures(none) [[AUR:%.*]]) {
+; UNROLL-NO-VF-NEXT: [[ENTRY:.*:]]
+; UNROLL-NO-VF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; UNROLL-NO-VF: [[VECTOR_MEMCHECK]]:
+; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD]], i64 512
+; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[AUD]], i64 512
+; UNROLL-NO-VF-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[ASR]], i64 512
+; UNROLL-NO-VF-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[AUR]], i64 512
; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ASD]], [[SCEVGEP1]]
; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[AUD]], [[SCEVGEP]]
; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -182,89 +184,89 @@ define void @test(ptr nocapture %asd, ptr nocapture %aud,
; UNROLL-NO-VF-NEXT: [[BOUND120:%.*]] = icmp ult ptr [[AUR]], [[SCEVGEP2]]
; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT21:%.*]] = and i1 [[BOUND019]], [[BOUND120]]
; UNROLL-NO-VF-NEXT: [[CONFLICT_RDX22:%.*]] = or i1 [[CONFLICT_RDX18]], [[FOUND_CONFLICT21]]
-; UNROLL-NO-VF-NEXT: br i1 [[CONFLICT_RDX22]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL-NO-VF: vector.ph:
-; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NO-VF: vector.body:
-; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE24:%.*]] ]
-; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[TMP1]]
-; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[TMP1]]
-; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[TMP1]]
-; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[AUR]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[AUR]], i64 [[TMP1]]
-; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP2]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
-; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP3]], align 4, !alias.scope [[META5]], !noalias [[META8]]
-; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
-; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META12]], !noalias [[META13]]
-; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META14:![0-9]+]], !noalias [[META15:![0-9]+]]
-; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META14]], !noalias [[META15]]
+; UNROLL-NO-VF-NEXT: br i1 [[CONFLICT_RDX22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NO-VF: [[VECTOR_PH]]:
+; UNROLL-NO-VF-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NO-VF: [[VECTOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE24:.*]] ]
+; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[AUR]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[AUR]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP1]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
+; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP2]], align 4, !alias.scope [[META5]], !noalias [[META8]]
+; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP3]], align 4, !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
+; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META12]], !noalias [[META13]]
+; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META14:![0-9]+]], !noalias [[META15:![0-9]+]]
+; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META14]], !noalias [[META15]]
+; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META15]]
; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4, !alias.scope [[META15]]
-; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP9]], align 4, !alias.scope [[META15]]
+; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = add nsw i32 [[TMP9]], 23
; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = add nsw i32 [[TMP10]], 23
-; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = add nsw i32 [[TMP11]], 23
+; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = add nsw i32 [[TMP11]], 24
; UNROLL-NO-VF-NEXT: [[TMP20:%.*]] = add nsw i32 [[TMP12]], 24
-; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = add nsw i32 [[TMP13]], 24
+; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = add nsw i32 [[TMP13]], 25
; UNROLL-NO-VF-NEXT: [[TMP22:%.*]] = add nsw i32 [[TMP14]], 25
-; UNROLL-NO-VF-NEXT: [[TMP23:%.*]] = add nsw i32 [[TMP15]], 25
+; UNROLL-NO-VF-NEXT: [[TMP23:%.*]] = add nsw i32 [[TMP15]], 26
; UNROLL-NO-VF-NEXT: [[TMP24:%.*]] = add nsw i32 [[TMP16]], 26
-; UNROLL-NO-VF-NEXT: [[TMP25:%.*]] = add nsw i32 [[TMP17]], 26
+; UNROLL-NO-VF-NEXT: [[TMP25:%.*]] = icmp slt i32 [[TMP9]], 100
; UNROLL-NO-VF-NEXT: [[TMP26:%.*]] = icmp slt i32 [[TMP10]], 100
-; UNROLL-NO-VF-NEXT: [[TMP27:%.*]] = icmp slt i32 [[TMP11]], 100
-; UNROLL-NO-VF-NEXT: br i1 [[TMP26]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
-; UNROLL-NO-VF: pred.urem.if:
-; UNROLL-NO-VF-NEXT: [[TMP28:%.*]] = sdiv i32 [[TMP18]], [[TMP10]]
-; UNROLL-NO-VF-NEXT: [[TMP29:%.*]] = udiv i32 [[TMP20]], [[TMP12]]
-; UNROLL-NO-VF-NEXT: [[TMP30:%.*]] = srem i32 [[TMP22]], [[TMP14]]
-; UNROLL-NO-VF-NEXT: [[TMP31:%.*]] = urem i32 [[TMP24]], [[TMP16]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_UREM_CONTINUE]]
-; UNROLL-NO-VF: pred.urem.continue:
-; UNROLL-NO-VF-NEXT: [[TMP32:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP28]], [[PRED_UREM_IF]] ]
-; UNROLL-NO-VF-NEXT: [[TMP33:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP29]], [[PRED_UREM_IF]] ]
-; UNROLL-NO-VF-NEXT: [[TMP34:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP30]], [[PRED_UREM_IF]] ]
-; UNROLL-NO-VF-NEXT: [[TMP35:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP31]], [[PRED_UREM_IF]] ]
-; UNROLL-NO-VF-NEXT: br i1 [[TMP27]], label [[PRED_UREM_IF23:%.*]], label [[PRED_UREM_CONTINUE24]]
-; UNROLL-NO-VF: pred.urem.if23:
-; UNROLL-NO-VF-NEXT: [[TMP36:%.*]] = sdiv i32 [[TMP19]], [[TMP11]]
-; UNROLL-NO-VF-NEXT: [[TMP37:%.*]] = udiv i32 [[TMP21]], [[TMP13]]
-; UNROLL-NO-VF-NEXT: [[TMP38:%.*]] = srem i32 [[TMP23]], [[TMP15]]
-; UNROLL-NO-VF-NEXT: [[TMP39:%.*]] = urem i32 [[TMP25]], [[TMP17]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_UREM_CONTINUE24]]
-; UNROLL-NO-VF: pred.urem.continue24:
-; UNROLL-NO-VF-NEXT: [[TMP40:%.*]] = phi i32 [ poison, [[PRED_UREM_CONTINUE]] ], [ [[TMP36]], [[PRED_UREM_IF23]] ]
-; UNROLL-NO-VF-NEXT: [[TMP41:%.*]] = phi i32 [ poison, [[PRED_UREM_CONTINUE]] ], [ [[TMP37]], [[PRED_UREM_IF23]] ]
-; UNROLL-NO-VF-NEXT: [[TMP42:%.*]] = phi i32 [ poison, [[PRED_UREM_CONTINUE]] ], [ [[TMP38]], [[PRED_UREM_IF23]] ]
-; UNROLL-NO-VF-NEXT: [[TMP43:%.*]] = phi i32 [ poison, [[PRED_UREM_CONTINUE]] ], [ [[TMP39]], [[PRED_UREM_IF23]] ]
-; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP26]], i32 [[TMP32]], i32 [[TMP18]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI25:%.*]] = select i1 [[TMP27]], i32 [[TMP40]], i32 [[TMP19]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI26:%.*]] = select i1 [[TMP26]], i32 [[TMP33]], i32 [[TMP20]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI27:%.*]] = select i1 [[TMP27]], i32 [[TMP41]], i32 [[TMP21]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI28:%.*]] = select i1 [[TMP26]], i32 [[TMP34]], i32 [[TMP22]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI29:%.*]] = select i1 [[TMP27]], i32 [[TMP42]], i32 [[TMP23]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI30:%.*]] = select i1 [[TMP26]], i32 [[TMP35]], i32 [[TMP24]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI31:%.*]] = select i1 [[TMP27]], i32 [[TMP43]], i32 [[TMP25]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI]], ptr [[TMP2]], align 4, !alias.scope [[META5]], !noalias [[META8]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI25]], ptr [[TMP3]], align 4, !alias.scope [[META5]], !noalias [[META8]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI26]], ptr [[TMP4]], align 4, !alias.scope [[META12]], !noalias [[META13]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI27]], ptr [[TMP5]], align 4, !alias.scope [[META12]], !noalias [[META13]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI28]], ptr [[TMP6]], align 4, !alias.scope [[META14]], !noalias [[META15]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI29]], ptr [[TMP7]], align 4, !alias.scope [[META14]], !noalias [[META15]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI30]], ptr [[TMP8]], align 4, !alias.scope [[META15]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI31]], ptr [[TMP9]], align 4, !alias.scope [[META15]]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP25]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE:.*]]
+; UNROLL-NO-VF: [[PRED_UREM_IF]]:
+; UNROLL-NO-VF-NEXT: [[TMP27:%.*]] = sdiv i32 [[TMP17]], [[TMP9]]
+; UNROLL-NO-VF-NEXT: [[TMP28:%.*]] = udiv i32 [[TMP19]], [[TMP11]]
+; UNROLL-NO-VF-NEXT: [[TMP29:%.*]] = srem i32 [[TMP21]], [[TMP13]]
+; UNROLL-NO-VF-NEXT: [[TMP30:%.*]] = urem i32 [[TMP23]], [[TMP15]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_UREM_CONTINUE]]
+; UNROLL-NO-VF: [[PRED_UREM_CONTINUE]]:
+; UNROLL-NO-VF-NEXT: [[TMP31:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP27]], %[[PRED_UREM_IF]] ]
+; UNROLL-NO-VF-NEXT: [[TMP32:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP28]], %[[PRED_UREM_IF]] ]
+; UNROLL-NO-VF-NEXT: [[TMP33:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP29]], %[[PRED_UREM_IF]] ]
+; UNROLL-NO-VF-NEXT: [[TMP34:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP30]], %[[PRED_UREM_IF]] ]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP26]], label %[[PRED_UREM_IF23:.*]], label %[[PRED_UREM_CONTINUE24]]
+; UNROLL-NO-VF: [[PRED_UREM_IF23]]:
+; UNROLL-NO-VF-NEXT: [[TMP35:%.*]] = sdiv i32 [[TMP18]], [[TMP10]]
+; UNROLL-NO-VF-NEXT: [[TMP36:%.*]] = udiv i32 [[TMP20]], [[TMP12]]
+; UNROLL-NO-VF-NEXT: [[TMP37:%.*]] = srem i32 [[TMP22]], [[TMP14]]
+; UNROLL-NO-VF-NEXT: [[TMP38:%.*]] = urem i32 [[TMP24]], [[TMP16]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_UREM_CONTINUE24]]
+; UNROLL-NO-VF: [[PRED_UREM_CONTINUE24]]:
+; UNROLL-NO-VF-NEXT: [[TMP39:%.*]] = phi i32 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP35]], %[[PRED_UREM_IF23]] ]
+; UNROLL-NO-VF-NEXT: [[TMP40:%.*]] = phi i32 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP36]], %[[PRED_UREM_IF23]] ]
+; UNROLL-NO-VF-NEXT: [[TMP41:%.*]] = phi i32 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP37]], %[[PRED_UREM_IF23]] ]
+; UNROLL-NO-VF-NEXT: [[TMP42:%.*]] = phi i32 [ poison, %[[PRED_UREM_CONTINUE]] ], [ [[TMP38]], %[[PRED_UREM_IF23]] ]
+; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP25]], i32 [[TMP31]], i32 [[TMP17]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI25:%.*]] = select i1 [[TMP26]], i32 [[TMP39]], i32 [[TMP18]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI26:%.*]] = select i1 [[TMP25]], i32 [[TMP32]], i32 [[TMP19]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI27:%.*]] = select i1 [[TMP26]], i32 [[TMP40]], i32 [[TMP20]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI28:%.*]] = select i1 [[TMP25]], i32 [[TMP33]], i32 [[TMP21]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI29:%.*]] = select i1 [[TMP26]], i32 [[TMP41]], i32 [[TMP22]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI30:%.*]] = select i1 [[TMP25]], i32 [[TMP34]], i32 [[TMP23]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI31:%.*]] = select i1 [[TMP26]], i32 [[TMP42]], i32 [[TMP24]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI]], ptr [[TMP1]], align 4, !alias.scope [[META5]], !noalias [[META8]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI25]], ptr [[TMP2]], align 4, !alias.scope [[META5]], !noalias [[META8]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI26]], ptr [[TMP3]], align 4, !alias.scope [[META12]], !noalias [[META13]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI27]], ptr [[TMP4]], align 4, !alias.scope [[META12]], !noalias [[META13]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI28]], ptr [[TMP5]], align 4, !alias.scope [[META14]], !noalias [[META15]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI29]], ptr [[TMP6]], align 4, !alias.scope [[META14]], !noalias [[META15]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI30]], ptr [[TMP7]], align 4, !alias.scope [[META15]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI31]], ptr [[TMP8]], align 4, !alias.scope [[META15]]
; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT: [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; UNROLL-NO-VF-NEXT: br i1 [[TMP46]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; UNROLL-NO-VF: middle.block:
-; UNROLL-NO-VF-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
-; UNROLL-NO-VF: scalar.ph:
-; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]]
-; UNROLL-NO-VF: for.cond.cleanup:
+; UNROLL-NO-VF-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; UNROLL-NO-VF-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; UNROLL-NO-VF: [[MIDDLE_BLOCK]]:
+; UNROLL-NO-VF-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; UNROLL-NO-VF: [[SCALAR_PH]]:
+; UNROLL-NO-VF-NEXT: br label %[[FOR_BODY:.*]]
+; UNROLL-NO-VF: [[FOR_COND_CLEANUP]]:
; UNROLL-NO-VF-NEXT: ret void
-; UNROLL-NO-VF: for.body:
-; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ]
+; UNROLL-NO-VF: [[FOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ]
; UNROLL-NO-VF-NEXT: [[ISD:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDVARS_IV]]
; UNROLL-NO-VF-NEXT: [[IUD:%.*]] = getelementptr inbounds i32, ptr [[AUD]], i64 [[INDVARS_IV]]
; UNROLL-NO-VF-NEXT: [[ISR:%.*]] = getelementptr inbounds i32, ptr [[ASR]], i64 [[INDVARS_IV]]
@@ -278,25 +280,25 @@ define void @test(ptr nocapture %asd, ptr nocapture %aud,
; UNROLL-NO-VF-NEXT: [[PSR:%.*]] = add nsw i32 [[LSR]], 25
; UNROLL-NO-VF-NEXT: [[PUR:%.*]] = add nsw i32 [[LUR]], 26
; UNROLL-NO-VF-NEXT: [[CMP1:%.*]] = icmp slt i32 [[LSD]], 100
-; UNROLL-NO-VF-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END]]
-; UNROLL-NO-VF: if.then:
+; UNROLL-NO-VF-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_END]]
+; UNROLL-NO-VF: [[IF_THEN]]:
; UNROLL-NO-VF-NEXT: [[RSD:%.*]] = sdiv i32 [[PSD]], [[LSD]]
; UNROLL-NO-VF-NEXT: [[RUD:%.*]] = udiv i32 [[PUD]], [[LUD]]
; UNROLL-NO-VF-NEXT: [[RSR:%.*]] = srem i32 [[PSR]], [[LSR]]
; UNROLL-NO-VF-NEXT: [[RUR:%.*]] = urem i32 [[PUR]], [[LUR]]
-; UNROLL-NO-VF-NEXT: br label [[IF_END]]
-; UNROLL-NO-VF: if.end:
-; UNROLL-NO-VF-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], [[IF_THEN]] ], [ [[PSD]], [[FOR_BODY]] ]
-; UNROLL-NO-VF-NEXT: [[YUD_0:%.*]] = phi i32 [ [[RUD]], [[IF_THEN]] ], [ [[PUD]], [[FOR_BODY]] ]
-; UNROLL-NO-VF-NEXT: [[YSR_0:%.*]] = phi i32 [ [[RSR]], [[IF_THEN]] ], [ [[PSR]], [[FOR_BODY]] ]
-; UNROLL-NO-VF-NEXT: [[YUR_0:%.*]] = phi i32 [ [[RUR]], [[IF_THEN]] ], [ [[PUR]], [[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: br label %[[IF_END]]
+; UNROLL-NO-VF: [[IF_END]]:
+; UNROLL-NO-VF-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], %[[IF_THEN]] ], [ [[PSD]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[YUD_0:%.*]] = phi i32 [ [[RUD]], %[[IF_THEN]] ], [ [[PUD]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[YSR_0:%.*]] = phi i32 [ [[RSR]], %[[IF_THEN]] ], [ [[PSR]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[YUR_0:%.*]] = phi i32 [ [[RUR]], %[[IF_THEN]] ], [ [[PUR]], %[[FOR_BODY]] ]
; UNROLL-NO-VF-NEXT: store i32 [[YSD_0]], ptr [[ISD]], align 4
; UNROLL-NO-VF-NEXT: store i32 [[YUD_0]], ptr [[IUD]], align 4
; UNROLL-NO-VF-NEXT: store i32 [[YSR_0]], ptr [[ISR]], align 4
; UNROLL-NO-VF-NEXT: store i32 [[YUR_0]], ptr [[IUR]], align 4
; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 128
-; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
;
ptr nocapture %asr, ptr nocapture %aur) {
entry:
@@ -344,146 +346,148 @@ if.end:
}
define void @test_scalar2scalar(ptr nocapture %asd, ptr nocapture %bsd) {
-; CHECK-LABEL: @test_scalar2scalar(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD:%.*]], i64 512
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD:%.*]], i64 512
+; CHECK-LABEL: define void @test_scalar2scalar(
+; CHECK-SAME: ptr captures(none) [[ASD:%.*]], ptr captures(none) [[BSD:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD]], i64 512
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD]], i64 512
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ASD]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[BSD]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[PRED_SDIV_CONTINUE4:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META23:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META23]]
-; CHECK-NEXT: [[TMP5:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 23)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
-; CHECK: pred.sdiv.if:
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = sdiv i32 [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = sdiv i32 [[TMP11]], [[TMP10]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> poison, i32 [[TMP12]], i64 0
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE]]
-; CHECK: pred.sdiv.continue:
-; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP6]], i64 1
-; CHECK-NEXT: br i1 [[TMP16]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4]]
-; CHECK: pred.sdiv.if3:
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP17]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
-; CHECK-NEXT: [[TMP21:%.*]] = sdiv i32 [[TMP20]], [[TMP19]]
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP21]], i64 1
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE4]]
-; CHECK: pred.sdiv.continue4:
-; CHECK-NEXT: [[TMP24:%.*]] = phi <2 x i32> [ [[TMP15]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP22]], [[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP6]], <2 x i32> [[TMP24]], <2 x i32> [[TMP5]]
-; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP1]], align 4, !alias.scope [[META20]], !noalias [[META23]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE4:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META23:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META23]]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 23)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK: [[PRED_SDIV_IF]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = sdiv i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK: [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP10]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
+; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4]]
+; CHECK: [[PRED_SDIV_IF3]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
+; CHECK-NEXT: [[TMP17:%.*]] = sdiv i32 [[TMP16]], [[TMP15]]
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP17]], i64 1
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
+; CHECK: [[PRED_SDIV_CONTINUE4]]:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <2 x i32> [ [[TMP11]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP18]], %[[PRED_SDIV_IF3]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP19]], <2 x i32> [[TMP2]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP0]], align 4, !alias.scope [[META20]], !noalias [[META23]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP27]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
-; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP20]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
-; CHECK: for.body:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ], [ 0, [[ENTRY]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[ISD:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[LSD:%.*]] = load i32, ptr [[ISD]], align 4
; CHECK-NEXT: [[ISD_B:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[LSD_B:%.*]] = load i32, ptr [[ISD_B]], align 4
; CHECK-NEXT: [[PSD:%.*]] = add nsw i32 [[LSD]], 23
; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[LSD]], 100
-; CHECK-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END]]
-; CHECK: if.then:
+; CHECK-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_END]]
+; CHECK: [[IF_THEN]]:
; CHECK-NEXT: [[SD1:%.*]] = sdiv i32 [[PSD]], [[LSD]]
; CHECK-NEXT: [[RSD:%.*]] = sdiv i32 [[LSD_B]], [[SD1]]
-; CHECK-NEXT: br label [[IF_END]]
-; CHECK: if.end:
-; CHECK-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], [[IF_THEN]] ], [ [[PSD]], [[FOR_BODY]] ]
+; CHECK-NEXT: br label %[[IF_END]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], %[[IF_THEN]] ], [ [[PSD]], %[[FOR_BODY]] ]
; CHECK-NEXT: store i32 [[YSD_0]], ptr [[ISD]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
;
-; UNROLL-NO-VF-LABEL: @test_scalar2scalar(
-; UNROLL-NO-VF-NEXT: entry:
-; UNROLL-NO-VF-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
-; UNROLL-NO-VF: vector.memcheck:
-; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD:%.*]], i64 512
-; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD:%.*]], i64 512
+; UNROLL-NO-VF-LABEL: define void @test_scalar2scalar(
+; UNROLL-NO-VF-SAME: ptr captures(none) [[ASD:%.*]], ptr captures(none) [[BSD:%.*]]) {
+; UNROLL-NO-VF-NEXT: [[ENTRY:.*:]]
+; UNROLL-NO-VF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; UNROLL-NO-VF: [[VECTOR_MEMCHECK]]:
+; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD]], i64 512
+; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD]], i64 512
; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ASD]], [[SCEVGEP1]]
; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[BSD]], [[SCEVGEP]]
; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL-NO-VF: vector.ph:
-; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NO-VF: vector.body:
-; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SDIV_CONTINUE3:%.*]] ]
-; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[TMP1]]
-; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META23:![0-9]+]]
-; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4, !alias.scope [[META20]], !noalias [[META23]]
-; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[TMP1]]
+; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NO-VF: [[VECTOR_PH]]:
+; UNROLL-NO-VF-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NO-VF: [[VECTOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE3:.*]] ]
+; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META23:![0-9]+]]
+; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4, !alias.scope [[META20]], !noalias [[META23]]
+; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META23]]
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META23]]
-; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META23]]
+; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP3]], 23
; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = add nsw i32 [[TMP4]], 23
-; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = add nsw i32 [[TMP5]], 23
+; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP3]], 100
; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = icmp slt i32 [[TMP4]], 100
-; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP5]], 100
-; UNROLL-NO-VF-NEXT: br i1 [[TMP12]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
-; UNROLL-NO-VF: pred.sdiv.if:
-; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = sdiv i32 [[TMP10]], [[TMP4]]
-; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP8]], [[TMP14]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_SDIV_CONTINUE]]
-; UNROLL-NO-VF: pred.sdiv.continue:
-; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP15]], [[PRED_SDIV_IF]] ]
-; UNROLL-NO-VF-NEXT: br i1 [[TMP13]], label [[PRED_SDIV_IF2:%.*]], label [[PRED_SDIV_CONTINUE3]]
-; UNROLL-NO-VF: pred.sdiv.if2:
-; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = sdiv i32 [[TMP11]], [[TMP5]]
-; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP9]], [[TMP18]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_SDIV_CONTINUE3]]
-; UNROLL-NO-VF: pred.sdiv.continue3:
-; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP19]], [[PRED_SDIV_IF2]] ]
-; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP12]], i32 [[TMP17]], i32 [[TMP10]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP13]], i32 [[TMP21]], i32 [[TMP11]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI]], ptr [[TMP2]], align 4, !alias.scope [[META20]], !noalias [[META23]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI4]], ptr [[TMP3]], align 4, !alias.scope [[META20]], !noalias [[META23]]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP11]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; UNROLL-NO-VF: [[PRED_SDIV_IF]]:
+; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = sdiv i32 [[TMP9]], [[TMP3]]
+; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = sdiv i32 [[TMP7]], [[TMP13]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; UNROLL-NO-VF: [[PRED_SDIV_CONTINUE]]:
+; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP14]], %[[PRED_SDIV_IF]] ]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP12]], label %[[PRED_SDIV_IF2:.*]], label %[[PRED_SDIV_CONTINUE3]]
+; UNROLL-NO-VF: [[PRED_SDIV_IF2]]:
+; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = sdiv i32 [[TMP10]], [[TMP4]]
+; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = sdiv i32 [[TMP8]], [[TMP16]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_SDIV_CONTINUE3]]
+; UNROLL-NO-VF: [[PRED_SDIV_CONTINUE3]]:
+; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = phi i32 [ poison, %[[PRED_SDIV_CONTINUE]] ], [ [[TMP17]], %[[PRED_SDIV_IF2]] ]
+; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP11]], i32 [[TMP15]], i32 [[TMP9]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP12]], i32 [[TMP18]], i32 [[TMP10]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI]], ptr [[TMP1]], align 4, !alias.scope [[META20]], !noalias [[META23]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI4]], ptr [[TMP2]], align 4, !alias.scope [[META20]], !noalias [[META23]]
; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; UNROLL-NO-VF-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
-; UNROLL-NO-VF: middle.block:
-; UNROLL-NO-VF-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
-; UNROLL-NO-VF: scalar.ph:
-; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]]
-; UNROLL-NO-VF: for.cond.cleanup:
+; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; UNROLL-NO-VF-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; UNROLL-NO-VF: [[MIDDLE_BLOCK]]:
+; UNROLL-NO-VF-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; UNROLL-NO-VF: [[SCALAR_PH]]:
+; UNROLL-NO-VF-NEXT: br label %[[FOR_BODY:.*]]
+; UNROLL-NO-VF: [[FOR_COND_CLEANUP]]:
; UNROLL-NO-VF-NEXT: ret void
-; UNROLL-NO-VF: for.body:
-; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ]
+; UNROLL-NO-VF: [[FOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ]
; UNROLL-NO-VF-NEXT: [[ISD:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDVARS_IV]]
; UNROLL-NO-VF-NEXT: [[LSD:%.*]] = load i32, ptr [[ISD]], align 4
; UNROLL-NO-VF-NEXT: [[ISD_B:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDVARS_IV]]
; UNROLL-NO-VF-NEXT: [[LSD_B:%.*]] = load i32, ptr [[ISD_B]], align 4
; UNROLL-NO-VF-NEXT: [[PSD:%.*]] = add nsw i32 [[LSD]], 23
; UNROLL-NO-VF-NEXT: [[CMP1:%.*]] = icmp slt i32 [[LSD]], 100
-; UNROLL-NO-VF-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END]]
-; UNROLL-NO-VF: if.then:
+; UNROLL-NO-VF-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_END]]
+; UNROLL-NO-VF: [[IF_THEN]]:
; UNROLL-NO-VF-NEXT: [[SD1:%.*]] = sdiv i32 [[PSD]], [[LSD]]
; UNROLL-NO-VF-NEXT: [[RSD:%.*]] = sdiv i32 [[LSD_B]], [[SD1]]
-; UNROLL-NO-VF-NEXT: br label [[IF_END]]
-; UNROLL-NO-VF: if.end:
-; UNROLL-NO-VF-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], [[IF_THEN]] ], [ [[PSD]], [[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: br label %[[IF_END]]
+; UNROLL-NO-VF: [[IF_END]]:
+; UNROLL-NO-VF-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], %[[IF_THEN]] ], [ [[PSD]], %[[FOR_BODY]] ]
; UNROLL-NO-VF-NEXT: store i32 [[YSD_0]], ptr [[ISD]], align 4
; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 128
-; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
;
entry:
br label %for.body
@@ -516,59 +520,60 @@ if.end:
}
define void @pr30172(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 {;
-; CHECK-LABEL: @pr30172(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD:%.*]], i64 512
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD:%.*]], i64 512
+; CHECK-LABEL: define void @pr30172(
+; CHECK-SAME: ptr captures(none) [[ASD:%.*]], ptr captures(none) [[BSD:%.*]]) !dbg [[DBG27:![0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD]], i64 512
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD]], i64 512
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ASD]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[BSD]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[PRED_SDIV_CONTINUE4:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META29:![0-9]+]], !noalias [[META32:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META32]]
-; CHECK-NEXT: [[TMP5:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 23)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
-; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true), !dbg [[DBG34:![0-9]+]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], splat (i32 200)
-; CHECK-NEXT: [[TMP9:%.*]] = select <2 x i1> [[TMP4]], <2 x i1> [[TMP8]], <2 x i1> zeroinitializer, !dbg [[DBG35:![0-9]+]]
-; CHECK-NEXT: [[TMP10:%.*]] = or <2 x i1> [[TMP9]], [[TMP6]]
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
-; CHECK-NEXT: br i1 [[TMP11]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
-; CHECK: pred.sdiv.if:
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = sdiv i32 [[TMP12]], [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = sdiv i32 [[TMP15]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP16]], i64 0
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE]]
-; CHECK: pred.sdiv.continue:
-; CHECK-NEXT: [[TMP19:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP17]], [[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
-; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4]]
-; CHECK: pred.sdiv.if3:
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP23:%.*]] = sdiv i32 [[TMP21]], [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
-; CHECK-NEXT: [[TMP25:%.*]] = sdiv i32 [[TMP24]], [[TMP23]]
-; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP25]], i64 1
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE4]]
-; CHECK: pred.sdiv.continue4:
-; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i32> [ [[TMP19]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP26]], [[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP10]], <2 x i32> [[TMP28]], <2 x i32> [[TMP5]]
-; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP1]], align 4, !alias.scope [[META29]], !noalias [[META32]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE4:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META29:![0-9]+]], !noalias [[META32:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META32]]
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 23)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
+; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true), !dbg [[DBG34:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], splat (i32 200)
+; CHECK-NEXT: [[TMP6:%.*]] = select <2 x i1> [[TMP4]], <2 x i1> [[TMP5]], <2 x i1> zeroinitializer, !dbg [[DBG35:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = or <2 x i1> [[TMP6]], [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK: [[PRED_SDIV_IF]]:
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = sdiv i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = sdiv i32 [[TMP12]], [[TMP11]]
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> poison, i32 [[TMP13]], i64 0
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK: [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP14]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1
+; CHECK-NEXT: br i1 [[TMP16]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4]]
+; CHECK: [[PRED_SDIV_IF3]]:
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
+; CHECK-NEXT: [[TMP21:%.*]] = sdiv i32 [[TMP20]], [[TMP19]]
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP21]], i64 1
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
+; CHECK: [[PRED_SDIV_CONTINUE4]]:
+; CHECK-NEXT: [[TMP23:%.*]] = phi <2 x i32> [ [[TMP15]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP22]], %[[PRED_SDIV_IF3]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i32> [[TMP23]], <2 x i32> [[TMP2]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP0]], align 4, !alias.scope [[META29]], !noalias [[META32]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP32]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
-; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP24]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
-; CHECK: for.body:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ], [ 0, [[ENTRY]] ]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[ISD:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[LSD:%.*]] = load i32, ptr [[ISD]], align 4
; CHECK-NEXT: [[ISD_B:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDVARS_IV]]
@@ -577,102 +582,103 @@ define void @pr30172(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 {;
; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[LSD]], 100
; CHECK-NEXT: [[CMP2:%.*]] = icmp sge i32 [[LSD]], 200
; CHECK-NEXT: [[OR_COND:%.*]] = or i1 [[CMP1]], [[CMP2]], !dbg [[DBG34]]
-; CHECK-NEXT: br i1 [[OR_COND]], label [[IF_THEN:%.*]], label [[IF_END]], !dbg [[DBG34]]
-; CHECK: if.then:
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[IF_THEN:.*]], label %[[IF_END]], !dbg [[DBG34]]
+; CHECK: [[IF_THEN]]:
; CHECK-NEXT: [[SD1:%.*]] = sdiv i32 [[PSD]], [[LSD]]
; CHECK-NEXT: [[RSD:%.*]] = sdiv i32 [[LSD_B]], [[SD1]]
-; CHECK-NEXT: br label [[IF_END]]
-; CHECK: if.end:
-; CHECK-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], [[IF_THEN]] ], [ [[PSD]], [[FOR_BODY]] ]
+; CHECK-NEXT: br label %[[IF_END]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], %[[IF_THEN]] ], [ [[PSD]], %[[FOR_BODY]] ]
; CHECK-NEXT: store i32 [[YSD_0]], ptr [[ISD]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
;
-; UNROLL-NO-VF-LABEL: @pr30172(
-; UNROLL-NO-VF-NEXT: entry:
-; UNROLL-NO-VF-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
-; UNROLL-NO-VF: vector.memcheck:
-; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD:%.*]], i64 512
-; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD:%.*]], i64 512
+; UNROLL-NO-VF-LABEL: define void @pr30172(
+; UNROLL-NO-VF-SAME: ptr captures(none) [[ASD:%.*]], ptr captures(none) [[BSD:%.*]]) !dbg [[DBG27:![0-9]+]] {
+; UNROLL-NO-VF-NEXT: [[ENTRY:.*:]]
+; UNROLL-NO-VF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; UNROLL-NO-VF: [[VECTOR_MEMCHECK]]:
+; UNROLL-NO-VF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ASD]], i64 512
+; UNROLL-NO-VF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BSD]], i64 512
; UNROLL-NO-VF-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ASD]], [[SCEVGEP1]]
; UNROLL-NO-VF-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[BSD]], [[SCEVGEP]]
; UNROLL-NO-VF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL-NO-VF: vector.ph:
-; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NO-VF: vector.body:
-; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SDIV_CONTINUE3:%.*]] ]
-; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[TMP1]]
-; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4, !alias.scope [[META29:![0-9]+]], !noalias [[META32:![0-9]+]]
-; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4, !alias.scope [[META29]], !noalias [[META32]]
-; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[TMP1]]
+; UNROLL-NO-VF-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NO-VF: [[VECTOR_PH]]:
+; UNROLL-NO-VF-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NO-VF: [[VECTOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE3:.*]] ]
+; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4, !alias.scope [[META29:![0-9]+]], !noalias [[META32:![0-9]+]]
+; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4, !alias.scope [[META29]], !noalias [[META32]]
+; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META32]]
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META32]]
-; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META32]]
+; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP3]], 23
; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = add nsw i32 [[TMP4]], 23
-; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = add nsw i32 [[TMP5]], 23
+; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP3]], 100
; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = icmp slt i32 [[TMP4]], 100
-; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP5]], 100
-; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = xor i1 [[TMP12]], true, !dbg [[DBG34:![0-9]+]]
-; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true, !dbg [[DBG34]]
-; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = icmp sge i32 [[TMP4]], 200
-; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = icmp sge i32 [[TMP5]], 200
-; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = select i1 [[TMP17]], i1 [[TMP15]], i1 false, !dbg [[DBG35:![0-9]+]]
-; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], i1 [[TMP16]], i1 false, !dbg [[DBG35]]
+; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = xor i1 [[TMP11]], true, !dbg [[DBG34:![0-9]+]]
+; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = xor i1 [[TMP12]], true, !dbg [[DBG34]]
+; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = icmp sge i32 [[TMP3]], 200
+; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = icmp sge i32 [[TMP4]], 200
+; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = select i1 [[TMP13]], i1 [[TMP15]], i1 false, !dbg [[DBG35:![0-9]+]]
+; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], i1 [[TMP16]], i1 false, !dbg [[DBG35]]
+; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = or i1 [[TMP17]], [[TMP11]]
; UNROLL-NO-VF-NEXT: [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP12]]
-; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = or i1 [[TMP19]], [[TMP13]]
-; UNROLL-NO-VF-NEXT: br i1 [[TMP20]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
-; UNROLL-NO-VF: pred.sdiv.if:
-; UNROLL-NO-VF-NEXT: [[TMP22:%.*]] = sdiv i32 [[TMP10]], [[TMP4]]
-; UNROLL-NO-VF-NEXT: [[TMP23:%.*]] = sdiv i32 [[TMP8]], [[TMP22]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_SDIV_CONTINUE]]
-; UNROLL-NO-VF: pred.sdiv.continue:
-; UNROLL-NO-VF-NEXT: [[TMP25:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP23]], [[PRED_SDIV_IF]] ]
-; UNROLL-NO-VF-NEXT: br i1 [[TMP21]], label [[PRED_SDIV_IF2:%.*]], label [[PRED_SDIV_CONTINUE3]]
-; UNROLL-NO-VF: pred.sdiv.if2:
-; UNROLL-NO-VF-NEXT: [[TMP26:%.*]] = sdiv i32 [[TMP11]], [[TMP5]]
-; UNROLL-NO-VF-NEXT: [[TMP27:%.*]] = sdiv i32 [[TMP9]], [[TMP26]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_SDIV_CONTINUE3]]
-; UNROLL-NO-VF: pred.sdiv.continue3:
-; UNROLL-NO-VF-NEXT: [[TMP29:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP27]], [[PRED_SDIV_IF2]] ]
-; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP20]], i32 [[TMP25]], i32 [[TMP10]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP21]], i32 [[TMP29]], i32 [[TMP11]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI]], ptr [[TMP2]], align 4, !alias.scope [[META29]], !noalias [[META32]]
-; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI4]], ptr [[TMP3]], align 4, !alias.scope [[META29]], !noalias [[META32]]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP19]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; UNROLL-NO-VF: [[PRED_SDIV_IF]]:
+; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = sdiv i32 [[TMP9]], [[TMP3]]
+; UNROLL-NO-VF-NEXT: [[TMP22:%.*]] = sdiv i32 [[TMP7]], [[TMP21]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; UNROLL-NO-VF: [[PRED_SDIV_CONTINUE]]:
+; UNROLL-NO-VF-NEXT: [[TMP23:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP22]], %[[PRED_SDIV_IF]] ]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP20]], label %[[PRED_SDIV_IF2:.*]], label %[[PRED_SDIV_CONTINUE3]]
+; UNROLL-NO-VF: [[PRED_SDIV_IF2]]:
+; UNROLL-NO-VF-NEXT: [[TMP24:%.*]] = sdiv i32 [[TMP10]], [[TMP4]]
+; UNROLL-NO-VF-NEXT: [[TMP25:%.*]] = sdiv i32 [[TMP8]], [[TMP24]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_SDIV_CONTINUE3]]
+; UNROLL-NO-VF: [[PRED_SDIV_CONTINUE3]]:
+; UNROLL-NO-VF-NEXT: [[TMP26:%.*]] = phi i32 [ poison, %[[PRED_SDIV_CONTINUE]] ], [ [[TMP25]], %[[PRED_SDIV_IF2]] ]
+; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP19]], i32 [[TMP23]], i32 [[TMP9]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP20]], i32 [[TMP26]], i32 [[TMP10]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI]], ptr [[TMP1]], align 4, !alias.scope [[META29]], !noalias [[META32]]
+; UNROLL-NO-VF-NEXT: store i32 [[PREDPHI4]], ptr [[TMP2]], align 4, !alias.scope [[META29]], !noalias [[META32]]
; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; UNROLL-NO-VF-NEXT: br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
-; UNROLL-NO-VF: middle.block:
-; UNROLL-NO-VF-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
-; UNROLL-NO-VF: scalar.ph:
-; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]]
-; UNROLL-NO-VF: for.cond.cleanup:
+; UNROLL-NO-VF-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; UNROLL-NO-VF-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; UNROLL-NO-VF: [[MIDDLE_BLOCK]]:
+; UNROLL-NO-VF-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; UNROLL-NO-VF: [[SCALAR_PH]]:
+; UNROLL-NO-VF-NEXT: br label %[[FOR_BODY:.*]]
+; UNROLL-NO-VF: [[FOR_COND_CLEANUP]]:
; UNROLL-NO-VF-NEXT: ret void
-; UNROLL-NO-VF: for.body:
-; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ]
+; UNROLL-NO-VF: [[FOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END:.*]] ]
; UNROLL-NO-VF-NEXT: [[ISD:%.*]] = getelementptr inbounds i32, ptr [[ASD]], i64 [[INDVARS_IV]]
; UNROLL-NO-VF-NEXT: [[LSD:%.*]] = load i32, ptr [[ISD]], align 4
; UNROLL-NO-VF-NEXT: [[ISD_B:%.*]] = getelementptr inbounds i32, ptr [[BSD]], i64 [[INDVARS_IV]]
; UNROLL-NO-VF-NEXT: [[LSD_B:%.*]] = load i32, ptr [[ISD_B]], align 4
; UNROLL-NO-VF-NEXT: [[PSD:%.*]] = add nsw i32 [[LSD]], 23
; UNROLL-NO-VF-NEXT: [[CMP1:%.*]] = icmp slt i32 [[LSD]], 100
-; UNROLL-NO-VF-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[CHECKBB:%.*]], !dbg [[DBG34]]
-; UNROLL-NO-VF: checkbb:
+; UNROLL-NO-VF-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[CHECKBB:.*]], !dbg [[DBG34]]
+; UNROLL-NO-VF: [[CHECKBB]]:
; UNROLL-NO-VF-NEXT: [[CMP2:%.*]] = icmp sge i32 [[LSD]], 200
-; UNROLL-NO-VF-NEXT: br i1 [[CMP2]], label [[IF_THEN]], label [[IF_END]], !dbg [[DBG35]]
-; UNROLL-NO-VF: if.then:
+; UNROLL-NO-VF-NEXT: br i1 [[CMP2]], label %[[IF_THEN]], label %[[IF_END]], !dbg [[DBG35]]
+; UNROLL-NO-VF: [[IF_THEN]]:
; UNROLL-NO-VF-NEXT: [[SD1:%.*]] = sdiv i32 [[PSD]], [[LSD]]
; UNROLL-NO-VF-NEXT: [[RSD:%.*]] = sdiv i32 [[LSD_B]], [[SD1]]
-; UNROLL-NO-VF-NEXT: br label [[IF_END]]
-; UNROLL-NO-VF: if.end:
-; UNROLL-NO-VF-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], [[IF_THEN]] ], [ [[PSD]], [[CHECKBB]] ]
+; UNROLL-NO-VF-NEXT: br label %[[IF_END]]
+; UNROLL-NO-VF: [[IF_END]]:
+; UNROLL-NO-VF-NEXT: [[YSD_0:%.*]] = phi i32 [ [[RSD]], %[[IF_THEN]] ], [ [[PSD]], %[[CHECKBB]] ]
; UNROLL-NO-VF-NEXT: store i32 [[YSD_0]], ptr [[ISD]], align 4
; UNROLL-NO-VF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; UNROLL-NO-VF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 128
-; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; UNROLL-NO-VF-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
;
entry:
br label %for.body
@@ -708,138 +714,140 @@ if.end:
}
define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
-; CHECK-LABEL: @predicated_udiv_scalarized_operand(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; CHECK-LABEL: define i32 @predicated_udiv_scalarized_operand(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[C:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE2:%.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_UDIV_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP4]], [[X:%.*]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP6]], [[TMP5]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.if1:
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP11]], [[X]]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP14:%.*]] = udiv i32 [[TMP13]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP14]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.continue2:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i32> [ [[TMP9]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP15]], [[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP16]], <2 x i32> [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP18]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[PRED_UDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
+; CHECK: [[PRED_UDIV_IF]]:
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[TMP1]], [[X]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
+; CHECK: [[PRED_UDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
+; CHECK: [[PRED_UDIV_IF1]]:
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP7]], [[X]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP10]], i64 1
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
+; CHECK: [[PRED_UDIV_CONTINUE2]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP13]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP18]])
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP13]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T6:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[T6:%.*]], %[[FOR_INC]] ]
; CHECK-NEXT: [[T0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; CHECK-NEXT: [[T2:%.*]] = load i32, ptr [[T0]], align 4
-; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; CHECK: if.then:
+; CHECK-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; CHECK: [[IF_THEN]]:
; CHECK-NEXT: [[T3:%.*]] = add nsw i32 [[T2]], [[X]]
; CHECK-NEXT: [[T4:%.*]] = udiv i32 [[T2]], [[T3]]
-; CHECK-NEXT: br label [[FOR_INC]]
-; CHECK: for.inc:
-; CHECK-NEXT: [[T5:%.*]] = phi i32 [ [[T2]], [[FOR_BODY]] ], [ [[T4]], [[IF_THEN]] ]
+; CHECK-NEXT: br label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[T5:%.*]] = phi i32 [ [[T2]], %[[FOR_BODY]] ], [ [[T4]], %[[IF_THEN]] ]
; CHECK-NEXT: [[T6]] = add i32 [[R]], [[T5]]
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP39:![0-9]+]]
-; CHECK: for.end:
-; CHECK-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], [[FOR_INC]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], %[[FOR_INC]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[T7]]
;
-; UNROLL-NO-VF-LABEL: @predicated_udiv_scalarized_operand(
-; UNROLL-NO-VF-NEXT: entry:
-; UNROLL-NO-VF-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; UNROLL-NO-VF-LABEL: define i32 @predicated_udiv_scalarized_operand(
+; UNROLL-NO-VF-SAME: ptr [[A:%.*]], i1 [[C:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) {
+; UNROLL-NO-VF-NEXT: [[ENTRY:.*]]:
+; UNROLL-NO-VF-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2
-; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL-NO-VF: vector.ph:
+; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NO-VF: [[VECTOR_PH]]:
; UNROLL-NO-VF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 2
; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; UNROLL-NO-VF-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NO-VF: vector.body:
-; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE3:%.*]] ]
-; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[PRED_UDIV_CONTINUE3]] ]
-; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_UDIV_CONTINUE3]] ]
-; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; UNROLL-NO-VF-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NO-VF: [[VECTOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE3:.*]] ]
+; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[PRED_UDIV_CONTINUE3]] ]
+; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[PRED_UDIV_CONTINUE3]] ]
+; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4
; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4
-; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4
-; UNROLL-NO-VF-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; UNROLL-NO-VF: pred.udiv.if:
-; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = add nsw i32 [[TMP4]], [[X:%.*]]
-; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP4]], [[TMP6]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; UNROLL-NO-VF: pred.udiv.continue:
-; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ]
-; UNROLL-NO-VF-NEXT: br i1 [[C]], label [[PRED_UDIV_IF2:%.*]], label [[PRED_UDIV_CONTINUE3]]
-; UNROLL-NO-VF: pred.udiv.if2:
-; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP5]], [[X]]
-; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP5]], [[TMP9]]
-; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE3]]
-; UNROLL-NO-VF: pred.udiv.continue3:
-; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF2]] ]
-; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], i32 [[TMP8]], i32 [[TMP4]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[C]], i32 [[TMP11]], i32 [[TMP5]]
-; UNROLL-NO-VF-NEXT: [[TMP14]] = add i32 [[VEC_PHI]], [[PREDPHI]]
-; UNROLL-NO-VF-NEXT: [[TMP15]] = add i32 [[VEC_PHI1]], [[PREDPHI4]]
+; UNROLL-NO-VF-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
+; UNROLL-NO-VF: [[PRED_UDIV_IF]]:
+; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP3]], [[X]]
+; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 [[TMP3]], [[TMP5]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_UDIV_CONTINUE]]
+; UNROLL-NO-VF: [[PRED_UDIV_CONTINUE]]:
+; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_UDIV_IF]] ]
+; UNROLL-NO-VF-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF2:.*]], label %[[PRED_UDIV_CONTINUE3]]
+; UNROLL-NO-VF: [[PRED_UDIV_IF2]]:
+; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP4]], [[X]]
+; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP4]], [[TMP8]]
+; UNROLL-NO-VF-NEXT: br label %[[PRED_UDIV_CONTINUE3]]
+; UNROLL-NO-VF: [[PRED_UDIV_CONTINUE3]]:
+; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = phi i32 [ poison, %[[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_UDIV_IF2]] ]
+; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], i32 [[TMP7]], i32 [[TMP3]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[C]], i32 [[TMP10]], i32 [[TMP4]]
+; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI]], [[PREDPHI]]
+; UNROLL-NO-VF-NEXT: [[TMP12]] = add i32 [[VEC_PHI1]], [[PREDPHI4]]
; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NO-VF-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
-; UNROLL-NO-VF: middle.block:
-; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP15]], [[TMP14]]
+; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; UNROLL-NO-VF: [[MIDDLE_BLOCK]]:
+; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP12]], [[TMP11]]
; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; UNROLL-NO-VF: scalar.ph:
-; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; UNROLL-NO-VF-NEXT: br label [[FOR_BODY:%.*]]
-; UNROLL-NO-VF: for.body:
-; UNROLL-NO-VF-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; UNROLL-NO-VF-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T6:%.*]], [[FOR_INC]] ]
+; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; UNROLL-NO-VF: [[SCALAR_PH]]:
+; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; UNROLL-NO-VF-NEXT: br label %[[FOR_BODY:.*]]
+; UNROLL-NO-VF: [[FOR_BODY]]:
+; UNROLL-NO-VF-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; UNROLL-NO-VF-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[T6:%.*]], %[[FOR_INC]] ]
; UNROLL-NO-VF-NEXT: [[T0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; UNROLL-NO-VF-NEXT: [[T2:%.*]] = load i32, ptr [[T0]], align 4
-; UNROLL-NO-VF-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; UNROLL-NO-VF: if.then:
+; UNROLL-NO-VF-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; UNROLL-NO-VF: [[IF_THEN]]:
; UNROLL-NO-VF-NEXT: [[T3:%.*]] = add nsw i32 [[T2]], [[X]]
; UNROLL-NO-VF-NEXT: [[T4:%.*]] = udiv i32 [[T2]], [[T3]]
-; UNROLL-NO-VF-NEXT: br label [[FOR_INC]]
-; UNROLL-NO-VF: for.inc:
-; UNROLL-NO-VF-NEXT: [[T5:%.*]] = phi i32 [ [[T2]], [[FOR_BODY]] ], [ [[T4]], [[IF_THEN]] ]
+; UNROLL-NO-VF-NEXT: br label %[[FOR_INC]]
+; UNROLL-NO-VF: [[FOR_INC]]:
+; UNROLL-NO-VF-NEXT: [[T5:%.*]] = phi i32 [ [[T2]], %[[FOR_BODY]] ], [ [[T4]], %[[IF_THEN]] ]
; UNROLL-NO-VF-NEXT: [[T6]] = add i32 [[R]], [[T5]]
; UNROLL-NO-VF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; UNROLL-NO-VF-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; UNROLL-NO-VF-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP39:![0-9]+]]
-; UNROLL-NO-VF: for.end:
-; UNROLL-NO-VF-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], [[FOR_INC]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-VF-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP39:![0-9]+]]
+; UNROLL-NO-VF: [[FOR_END]]:
+; UNROLL-NO-VF-NEXT: [[T7:%.*]] = phi i32 [ [[T6]], %[[FOR_INC]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
; UNROLL-NO-VF-NEXT: ret i32 [[T7]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 9f17e3f34c9ac..db56646fe34b5 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -passes=loop-vectorize,simplifycfg -verify-loop-info -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck %s --check-prefix=UNROLL
; RUN: opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -passes=loop-vectorize -verify-loop-info < %s | FileCheck %s --check-prefix=UNROLL-NOSIMPLIFY
; RUN: opt -S -vectorize-num-stores-pred=1 -force-vector-width=2 -force-vector-interleave=1 -passes=loop-vectorize,simplifycfg -verify-loop-info -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck %s --check-prefix=VEC
@@ -7,100 +7,103 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
; Test predication of stores.
define i32 @test(ptr nocapture %f) #0 {
-; UNROLL-LABEL: @test(
-; UNROLL-NEXT: entry:
-; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL: vector.body:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; UNROLL-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[F:%.*]], i64 [[INDEX]]
-; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP1]]
+; UNROLL-LABEL: define i32 @test(
+; UNROLL-SAME: ptr captures(none) [[F:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL: [[VECTOR_BODY]]:
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[INDEX]]
+; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP0]]
+; UNROLL-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4
; UNROLL-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4
-; UNROLL-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4
+; UNROLL-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], 100
; UNROLL-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 100
-; UNROLL-NEXT: [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], 100
-; UNROLL-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; UNROLL: pred.store.if:
+; UNROLL-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL: [[PRED_STORE_IF]]:
+; UNROLL-NEXT: [[TMP7:%.*]] = add nsw i32 [[TMP3]], 20
+; UNROLL-NEXT: store i32 [[TMP7]], ptr [[TMP1]], align 4
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL: [[PRED_STORE_CONTINUE]]:
+; UNROLL-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_IF1]]:
; UNROLL-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP4]], 20
; UNROLL-NEXT: store i32 [[TMP8]], ptr [[TMP2]], align 4
-; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE]]
-; UNROLL: pred.store.continue:
-; UNROLL-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; UNROLL: pred.store.if1:
-; UNROLL-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP5]], 20
-; UNROLL-NEXT: store i32 [[TMP9]], ptr [[TMP3]], align 4
-; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; UNROLL: pred.store.continue2:
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; UNROLL-NEXT: br i1 [[TMP10]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; UNROLL: for.end:
+; UNROLL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; UNROLL-NEXT: br i1 [[TMP9]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UNROLL: [[FOR_END]]:
; UNROLL-NEXT: ret i32 0
;
-; UNROLL-NOSIMPLIFY-LABEL: @test(
-; UNROLL-NOSIMPLIFY-NEXT: entry:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: vector.ph:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NOSIMPLIFY: vector.body:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[F:%.*]], i64 [[INDEX]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP1]]
+; UNROLL-NOSIMPLIFY-LABEL: define i32 @test(
+; UNROLL-NOSIMPLIFY-SAME: ptr captures(none) [[F:%.*]]) {
+; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[INDEX]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP0]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], 100
; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 100
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; UNROLL-NOSIMPLIFY: pred.store.if:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = add nsw i32 [[TMP3]], 20
+; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP7]], ptr [[TMP1]], align 4
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP4]], 20
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP8]], ptr [[TMP2]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: pred.store.continue:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.if1:
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP5]], 20
-; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP9]], ptr [[TMP3]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.continue2:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; UNROLL-NOSIMPLIFY: middle.block:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[FOR_INC:%.*]]
-; UNROLL-NOSIMPLIFY: for.end:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[FOR_END:.*]]
+; UNROLL-NOSIMPLIFY: [[FOR_END]]:
; UNROLL-NOSIMPLIFY-NEXT: ret i32 0
;
-; VEC-LABEL: @test(
-; VEC-NEXT: entry:
-; VEC-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[F:%.*]], i64 [[INDEX]]
-; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; VEC-NEXT: [[TMP3:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
-; VEC-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; VEC: pred.store.if:
-; VEC-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP7]], 20
-; VEC-NEXT: store i32 [[TMP8]], ptr [[TMP1]], align 4
-; VEC-NEXT: br label [[PRED_STORE_CONTINUE]]
-; VEC: pred.store.continue:
-; VEC-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
-; VEC-NEXT: br i1 [[TMP9]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.if1:
-; VEC-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP10]]
-; VEC-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: [[TMP13:%.*]] = add nsw i32 [[TMP12]], 20
-; VEC-NEXT: store i32 [[TMP13]], ptr [[TMP11]], align 4
-; VEC-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.continue2:
+; VEC-LABEL: define i32 @test(
+; VEC-SAME: ptr captures(none) [[F:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[INDEX]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; VEC-NEXT: [[TMP1:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
+; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; VEC-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; VEC-NEXT: [[TMP4:%.*]] = add nsw i32 [[TMP3]], 20
+; VEC-NEXT: store i32 [[TMP4]], ptr [[TMP0]], align 4
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC: [[PRED_STORE_CONTINUE]]:
+; VEC-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; VEC-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_IF1]]:
+; VEC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP6]]
+; VEC-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; VEC-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], 20
+; VEC-NEXT: store i32 [[TMP9]], ptr [[TMP7]], align 4
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_CONTINUE2]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; VEC-NEXT: br i1 [[TMP14]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; VEC: for.end:
+; VEC-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; VEC-NEXT: br i1 [[TMP10]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VEC: [[FOR_END]]:
; VEC-NEXT: ret i32 0
;
entry:
@@ -135,214 +138,217 @@ for.end:
; PR18724
define void @bug18724(i1 %cond, ptr %ptr, i1 %cond.2, i64 %v.1, i32 %v.2) {
-; UNROLL-LABEL: @bug18724(
-; UNROLL-NEXT: entry:
-; UNROLL-NEXT: [[TMP0:%.*]] = xor i1 [[COND:%.*]], true
+; UNROLL-LABEL: define void @bug18724(
+; UNROLL-SAME: i1 [[COND:%.*]], ptr [[PTR:%.*]], i1 [[COND_2:%.*]], i64 [[V_1:%.*]], i32 [[V_2:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: [[TMP0:%.*]] = xor i1 [[COND]], true
; UNROLL-NEXT: call void @llvm.assume(i1 [[TMP0]])
-; UNROLL-NEXT: [[TMP1:%.*]] = trunc i64 [[V_1:%.*]] to i32
+; UNROLL-NEXT: [[TMP1:%.*]] = trunc i64 [[V_1]] to i32
; UNROLL-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 0)
; UNROLL-NEXT: [[TMP2:%.*]] = sub i32 [[SMAX]], [[TMP1]]
; UNROLL-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
; UNROLL-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 2
-; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL: vector.ph:
+; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 2
; UNROLL-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
-; UNROLL-NEXT: [[IND_END:%.*]] = add i64 [[V_1]], [[N_VEC]]
-; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL: vector.body:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
-; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi i32 [ [[V_2:%.*]], [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[PRED_STORE_CONTINUE3]] ]
-; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], [[PRED_STORE_CONTINUE3]] ]
+; UNROLL-NEXT: [[TMP5:%.*]] = add i64 [[V_1]], [[N_VEC]]
+; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL: [[VECTOR_BODY]]:
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi i32 [ [[V_2]], %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_STORE_CONTINUE3]] ]
+; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[PRED_STORE_CONTINUE3]] ]
; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[V_1]], [[INDEX]]
; UNROLL-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 1
-; UNROLL-NEXT: [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR:%.*]], i64 0, i64 [[OFFSET_IDX]]
+; UNROLL-NEXT: [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[OFFSET_IDX]]
; UNROLL-NEXT: [[TMP8:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP6]]
; UNROLL-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4
; UNROLL-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP8]], align 4
-; UNROLL-NEXT: br i1 [[COND_2:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE3]]
-; UNROLL: pred.store.if:
+; UNROLL-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE3]]
+; UNROLL: [[PRED_STORE_IF]]:
; UNROLL-NEXT: store i32 [[TMP9]], ptr [[TMP7]], align 4
; UNROLL-NEXT: store i32 [[TMP10]], ptr [[TMP8]], align 4
-; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE3]]
-; UNROLL: pred.store.continue3:
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; UNROLL: [[PRED_STORE_CONTINUE3]]:
; UNROLL-NEXT: [[TMP11:%.*]] = add i32 [[VEC_PHI]], 1
; UNROLL-NEXT: [[TMP12:%.*]] = add i32 [[VEC_PHI1]], 1
; UNROLL-NEXT: [[PREDPHI]] = select i1 [[COND_2]], i32 [[TMP11]], i32 [[VEC_PHI]]
; UNROLL-NEXT: [[PREDPHI4]] = select i1 [[COND_2]], i32 [[TMP12]], i32 [[VEC_PHI1]]
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; UNROLL: middle.block:
+; UNROLL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; UNROLL: [[MIDDLE_BLOCK]]:
; UNROLL-NEXT: [[BIN_RDX:%.*]] = add i32 [[PREDPHI4]], [[PREDPHI]]
; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
-; UNROLL-NEXT: [[TMP16:%.*]] = xor i1 [[CMP_N]], true
-; UNROLL-NEXT: call void @llvm.assume(i1 [[TMP16]])
-; UNROLL-NEXT: br label [[SCALAR_PH]]
-; UNROLL: scalar.ph:
-; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[V_1]], [[ENTRY:%.*]] ]
-; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ [[V_2]], [[ENTRY]] ]
-; UNROLL-NEXT: br label [[FOR_BODY14:%.*]]
-; UNROLL: for.body14:
-; UNROLL-NEXT: [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], [[FOR_INC23:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NEXT: [[INEWCHUNKS_120:%.*]] = phi i32 [ [[INEWCHUNKS_2:%.*]], [[FOR_INC23]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; UNROLL-NEXT: [[TMP14:%.*]] = xor i1 [[CMP_N]], true
+; UNROLL-NEXT: call void @llvm.assume(i1 [[TMP14]])
+; UNROLL-NEXT: br label %[[SCALAR_PH]]
+; UNROLL: [[SCALAR_PH]]:
+; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[V_1]], %[[ENTRY]] ]
+; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ [[V_2]], %[[ENTRY]] ]
+; UNROLL-NEXT: br label %[[FOR_BODY14:.*]]
+; UNROLL: [[FOR_BODY14]]:
+; UNROLL-NEXT: [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[FOR_INC23:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; UNROLL-NEXT: [[INEWCHUNKS_120:%.*]] = phi i32 [ [[INEWCHUNKS_2:%.*]], %[[FOR_INC23]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; UNROLL-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[INDVARS_IV3]]
; UNROLL-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX16]], align 4
-; UNROLL-NEXT: br i1 [[COND_2]], label [[IF_THEN18:%.*]], label [[FOR_INC23]]
-; UNROLL: if.then18:
+; UNROLL-NEXT: br i1 [[COND_2]], label %[[IF_THEN18:.*]], label %[[FOR_INC23]]
+; UNROLL: [[IF_THEN18]]:
; UNROLL-NEXT: store i32 [[TMP]], ptr [[ARRAYIDX16]], align 4
; UNROLL-NEXT: [[INC21:%.*]] = add nsw i32 [[INEWCHUNKS_120]], 1
-; UNROLL-NEXT: br label [[FOR_INC23]]
-; UNROLL: for.inc23:
-; UNROLL-NEXT: [[INEWCHUNKS_2]] = phi i32 [ [[INC21]], [[IF_THEN18]] ], [ [[INEWCHUNKS_120]], [[FOR_BODY14]] ]
+; UNROLL-NEXT: br label %[[FOR_INC23]]
+; UNROLL: [[FOR_INC23]]:
+; UNROLL-NEXT: [[INEWCHUNKS_2]] = phi i32 [ [[INC21]], %[[IF_THEN18]] ], [ [[INEWCHUNKS_120]], %[[FOR_BODY14]] ]
; UNROLL-NEXT: [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV3]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV3]] to i32
; UNROLL-NEXT: [[CMP13:%.*]] = icmp slt i32 [[TMP1]], 0
; UNROLL-NEXT: call void @llvm.assume(i1 [[CMP13]])
-; UNROLL-NEXT: br label [[FOR_BODY14]]
+; UNROLL-NEXT: br label %[[FOR_BODY14]]
;
-; UNROLL-NOSIMPLIFY-LABEL: @bug18724(
-; UNROLL-NOSIMPLIFY-NEXT: entry:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[FOR_BODY9:%.*]]
-; UNROLL-NOSIMPLIFY: for.body9:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND:%.*]], label [[FOR_INC26:%.*]], label [[FOR_BODY14_PREHEADER:%.*]]
-; UNROLL-NOSIMPLIFY: for.body14.preheader:
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = trunc i64 [[V_1:%.*]] to i32
+; UNROLL-NOSIMPLIFY-LABEL: define void @bug18724(
+; UNROLL-NOSIMPLIFY-SAME: i1 [[COND:%.*]], ptr [[PTR:%.*]], i1 [[COND_2:%.*]], i64 [[V_1:%.*]], i32 [[V_2:%.*]]) {
+; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[FOR_BODY9:.*]]
+; UNROLL-NOSIMPLIFY: [[FOR_BODY9]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND]], label %[[FOR_INC26:.*]], label %[[FOR_BODY14_PREHEADER:.*]]
+; UNROLL-NOSIMPLIFY: [[FOR_BODY14_PREHEADER]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = trunc i64 [[V_1]] to i32
; UNROLL-NOSIMPLIFY-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP0]], i32 0)
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sub i32 [[SMAX]], [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: vector.ph:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; UNROLL-NOSIMPLIFY-NEXT: [[IND_END:%.*]] = add i64 [[V_1]], [[N_VEC]]
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NOSIMPLIFY: vector.body:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: [[VEC_PHI:%.*]] = phi i32 [ [[V_2:%.*]], [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[PRED_STORE_CONTINUE3]] ]
-; UNROLL-NOSIMPLIFY-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], [[PRED_STORE_CONTINUE3]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = add i64 [[V_1]], [[N_VEC]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[VEC_PHI:%.*]] = phi i32 [ [[V_2]], %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_STORE_CONTINUE3]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[PRED_STORE_CONTINUE3]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[V_1]], [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 1
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR:%.*]], i64 0, i64 [[OFFSET_IDX]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[OFFSET_IDX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP5]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP6]], align 4
; UNROLL-NOSIMPLIFY-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND_2:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; UNROLL-NOSIMPLIFY: pred.store.if:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP8]], ptr [[TMP6]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: pred.store.continue:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND_2]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY: pred.store.if2:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP9]], ptr [[TMP7]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY: pred.store.continue3:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE3]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP10:%.*]] = add i32 [[VEC_PHI]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP11:%.*]] = add i32 [[VEC_PHI1]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[PREDPHI]] = select i1 [[COND_2]], i32 [[TMP10]], i32 [[VEC_PHI]]
; UNROLL-NOSIMPLIFY-NEXT: [[PREDPHI4]] = select i1 [[COND_2]], i32 [[TMP11]], i32 [[VEC_PHI1]]
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; UNROLL-NOSIMPLIFY: middle.block:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
; UNROLL-NOSIMPLIFY-NEXT: [[BIN_RDX:%.*]] = add i32 [[PREDPHI4]], [[PREDPHI]]
; UNROLL-NOSIMPLIFY-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[CMP_N]], label [[FOR_INC26_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; UNROLL-NOSIMPLIFY: scalar.ph:
-; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[V_1]], [[FOR_BODY14_PREHEADER]] ]
-; UNROLL-NOSIMPLIFY-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ [[V_2]], [[FOR_BODY14_PREHEADER]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br label [[FOR_BODY14:%.*]]
-; UNROLL-NOSIMPLIFY: for.body14:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], [[FOR_INC23:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_120:%.*]] = phi i32 [ [[INEWCHUNKS_2:%.*]], [[FOR_INC23]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[CMP_N]], label %[[FOR_INC26_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[V_1]], %[[FOR_BODY14_PREHEADER]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ [[V_2]], %[[FOR_BODY14_PREHEADER]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[FOR_BODY14:.*]]
+; UNROLL-NOSIMPLIFY: [[FOR_BODY14]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[FOR_INC23:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_120:%.*]] = phi i32 [ [[INEWCHUNKS_2:%.*]], %[[FOR_INC23]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[INDVARS_IV3]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX16]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND_2]], label [[IF_THEN18:%.*]], label [[FOR_INC23]]
-; UNROLL-NOSIMPLIFY: if.then18:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND_2]], label %[[IF_THEN18:.*]], label %[[FOR_INC23]]
+; UNROLL-NOSIMPLIFY: [[IF_THEN18]]:
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP]], ptr [[ARRAYIDX16]], align 4
; UNROLL-NOSIMPLIFY-NEXT: [[INC21:%.*]] = add nsw i32 [[INEWCHUNKS_120]], 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[FOR_INC23]]
-; UNROLL-NOSIMPLIFY: for.inc23:
-; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_2]] = phi i32 [ [[INC21]], [[IF_THEN18]] ], [ [[INEWCHUNKS_120]], [[FOR_BODY14]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[FOR_INC23]]
+; UNROLL-NOSIMPLIFY: [[FOR_INC23]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_2]] = phi i32 [ [[INC21]], %[[IF_THEN18]] ], [ [[INEWCHUNKS_120]], %[[FOR_BODY14]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV3]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV3]] to i32
; UNROLL-NOSIMPLIFY-NEXT: [[CMP13:%.*]] = icmp slt i32 [[TMP1]], 0
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[CMP13]], label [[FOR_BODY14]], label [[FOR_INC26_LOOPEXIT]], !llvm.loop [[LOOP4:![0-9]+]]
-; UNROLL-NOSIMPLIFY: for.inc26.loopexit:
-; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_2_LCSSA:%.*]] = phi i32 [ [[INEWCHUNKS_2]], [[FOR_INC23]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br label [[FOR_INC26]]
-; UNROLL-NOSIMPLIFY: for.inc26:
-; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_1_LCSSA:%.*]] = phi i32 [ 0, [[FOR_BODY9]] ], [ [[INEWCHUNKS_2_LCSSA]], [[FOR_INC26_LOOPEXIT]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[CMP13]], label %[[FOR_BODY14]], label %[[FOR_INC26_LOOPEXIT]], !llvm.loop [[LOOP4:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[FOR_INC26_LOOPEXIT]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_2_LCSSA:%.*]] = phi i32 [ [[INEWCHUNKS_2]], %[[FOR_INC23]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[FOR_INC26]]
+; UNROLL-NOSIMPLIFY: [[FOR_INC26]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INEWCHUNKS_1_LCSSA:%.*]] = phi i32 [ 0, %[[FOR_BODY9]] ], [ [[INEWCHUNKS_2_LCSSA]], %[[FOR_INC26_LOOPEXIT]] ]
; UNROLL-NOSIMPLIFY-NEXT: unreachable
;
-; VEC-LABEL: @bug18724(
-; VEC-NEXT: entry:
-; VEC-NEXT: [[TMP0:%.*]] = xor i1 [[COND:%.*]], true
+; VEC-LABEL: define void @bug18724(
+; VEC-SAME: i1 [[COND:%.*]], ptr [[PTR:%.*]], i1 [[COND_2:%.*]], i64 [[V_1:%.*]], i32 [[V_2:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[TMP0:%.*]] = xor i1 [[COND]], true
; VEC-NEXT: call void @llvm.assume(i1 [[TMP0]])
-; VEC-NEXT: [[TMP1:%.*]] = trunc i64 [[V_1:%.*]] to i32
+; VEC-NEXT: [[TMP1:%.*]] = trunc i64 [[V_1]] to i32
; VEC-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 0)
; VEC-NEXT: [[TMP2:%.*]] = sub i32 [[SMAX]], [[TMP1]]
; VEC-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
; VEC-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
; VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 2
-; VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; VEC: vector.ph:
+; VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC: [[VECTOR_PH]]:
; VEC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 2
; VEC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
-; VEC-NEXT: [[IND_END:%.*]] = add i64 [[V_1]], [[N_VEC]]
-; VEC-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[V_2:%.*]], i32 0
-; VEC-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; VEC-NEXT: [[TMP5:%.*]] = add i64 [[V_1]], [[N_VEC]]
+; VEC-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[V_2]], i32 0
+; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; VEC-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[V_1]], [[INDEX]]
-; VEC-NEXT: [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR:%.*]], i64 0, i64 [[OFFSET_IDX]]
+; VEC-NEXT: [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[OFFSET_IDX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 4
-; VEC-NEXT: br i1 [[COND_2:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.if:
-; VEC-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i32 [[TMP11]], ptr [[TMP7]], align 4
-; VEC-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 1
-; VEC-NEXT: [[TMP13:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP12]]
-; VEC-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i32 [[TMP14]], ptr [[TMP13]], align 4
-; VEC-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.continue2:
-; VEC-NEXT: [[TMP15:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
-; VEC-NEXT: [[PREDPHI]] = select i1 [[COND_2]], <2 x i32> [[TMP15]], <2 x i32> [[VEC_PHI]]
+; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; VEC-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4
+; VEC-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 1
+; VEC-NEXT: [[TMP10:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP9]]
+; VEC-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; VEC-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_CONTINUE2]]:
+; VEC-NEXT: [[TMP12:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
+; VEC-NEXT: [[PREDPHI]] = select i1 [[COND_2]], <2 x i32> [[TMP12]], <2 x i32> [[VEC_PHI]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; VEC: middle.block:
-; VEC-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI]])
+; VEC-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VEC: [[MIDDLE_BLOCK]]:
+; VEC-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI]])
; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
-; VEC-NEXT: [[TMP18:%.*]] = xor i1 [[CMP_N]], true
-; VEC-NEXT: call void @llvm.assume(i1 [[TMP18]])
-; VEC-NEXT: br label [[SCALAR_PH]]
-; VEC: scalar.ph:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[V_1]], [[ENTRY:%.*]] ]
-; VEC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ [[V_2]], [[ENTRY]] ]
-; VEC-NEXT: br label [[FOR_BODY14:%.*]]
-; VEC: for.body14:
-; VEC-NEXT: [[INDVARS_IV4:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], [[FOR_INC23:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC-NEXT: [[INEWCHUNKS_120:%.*]] = phi i32 [ [[INEWCHUNKS_2:%.*]], [[FOR_INC23]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; VEC-NEXT: [[ARRAYIDX17:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[INDVARS_IV4]]
-; VEC-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX17]], align 4
-; VEC-NEXT: br i1 [[COND_2]], label [[IF_THEN18:%.*]], label [[FOR_INC23]]
-; VEC: if.then18:
-; VEC-NEXT: store i32 [[TMP]], ptr [[ARRAYIDX17]], align 4
+; VEC-NEXT: [[TMP15:%.*]] = xor i1 [[CMP_N]], true
+; VEC-NEXT: call void @llvm.assume(i1 [[TMP15]])
+; VEC-NEXT: br label %[[SCALAR_PH]]
+; VEC: [[SCALAR_PH]]:
+; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[V_1]], %[[ENTRY]] ]
+; VEC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ [[V_2]], %[[ENTRY]] ]
+; VEC-NEXT: br label %[[FOR_BODY14:.*]]
+; VEC: [[FOR_BODY14]]:
+; VEC-NEXT: [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[FOR_INC23:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VEC-NEXT: [[INEWCHUNKS_120:%.*]] = phi i32 [ [[INEWCHUNKS_2:%.*]], %[[FOR_INC23]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; VEC-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[INDVARS_IV3]]
+; VEC-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX16]], align 4
+; VEC-NEXT: br i1 [[COND_2]], label %[[IF_THEN18:.*]], label %[[FOR_INC23]]
+; VEC: [[IF_THEN18]]:
+; VEC-NEXT: store i32 [[TMP]], ptr [[ARRAYIDX16]], align 4
; VEC-NEXT: [[INC21:%.*]] = add nsw i32 [[INEWCHUNKS_120]], 1
-; VEC-NEXT: br label [[FOR_INC23]]
-; VEC: for.inc23:
-; VEC-NEXT: [[INEWCHUNKS_2]] = phi i32 [ [[INC21]], [[IF_THEN18]] ], [ [[INEWCHUNKS_120]], [[FOR_BODY14]] ]
-; VEC-NEXT: [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV4]], 1
-; VEC-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV4]] to i32
+; VEC-NEXT: br label %[[FOR_INC23]]
+; VEC: [[FOR_INC23]]:
+; VEC-NEXT: [[INEWCHUNKS_2]] = phi i32 [ [[INC21]], %[[IF_THEN18]] ], [ [[INEWCHUNKS_120]], %[[FOR_BODY14]] ]
+; VEC-NEXT: [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV3]], 1
+; VEC-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV3]] to i32
; VEC-NEXT: [[CMP13:%.*]] = icmp slt i32 [[TMP1]], 0
; VEC-NEXT: call void @llvm.assume(i1 [[CMP13]])
-; VEC-NEXT: br label [[FOR_BODY14]]
+; VEC-NEXT: br label %[[FOR_BODY14]]
;
entry:
br label %for.body9
@@ -379,79 +385,82 @@ for.inc26:
; types, we shouldn't try to represent the expression in a smaller type.
;
define void @minimal_bit_widths(ptr %p, i1 %c) {
-; UNROLL-LABEL: @minimal_bit_widths(
-; UNROLL-NEXT: entry:
-; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL: vector.body:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; UNROLL-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[INDEX]]
-; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; UNROLL-LABEL: define void @minimal_bit_widths(
+; UNROLL-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL: [[VECTOR_BODY]]:
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
+; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
+; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1
-; UNROLL-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
-; UNROLL: pred.store.if:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_IF]]:
+; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: store i8 [[TMP5]], ptr [[TMP3]], align 1
-; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; UNROLL: pred.store.continue2:
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; UNROLL-NEXT: br i1 [[TMP6]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; UNROLL: for.end:
+; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; UNROLL-NEXT: br i1 [[TMP5]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; UNROLL: [[FOR_END]]:
; UNROLL-NEXT: ret void
;
-; UNROLL-NOSIMPLIFY-LABEL: @minimal_bit_widths(
-; UNROLL-NOSIMPLIFY-NEXT: entry:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: vector.ph:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NOSIMPLIFY: vector.body:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[INDEX]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; UNROLL-NOSIMPLIFY-LABEL: define void @minimal_bit_widths(
+; UNROLL-NOSIMPLIFY-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) {
+; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; UNROLL-NOSIMPLIFY: pred.store.if:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: pred.store.continue:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.if1:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP5]], ptr [[TMP3]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.continue2:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; UNROLL-NOSIMPLIFY: middle.block:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[FOR_INC:%.*]]
-; UNROLL-NOSIMPLIFY: for.end:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[FOR_END:.*]]
+; UNROLL-NOSIMPLIFY: [[FOR_END]]:
; UNROLL-NOSIMPLIFY-NEXT: ret void
;
-; VEC-LABEL: @minimal_bit_widths(
-; VEC-NEXT: entry:
-; VEC-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[INDEX]]
-; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP1]], align 1
-; VEC-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.if:
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP1]], align 1
-; VEC-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; VEC-NEXT: [[TMP7:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP7]], ptr [[TMP6]], align 1
-; VEC-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.continue2:
+; VEC-LABEL: define void @minimal_bit_widths(
+; VEC-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
+; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
+; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
+; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_CONTINUE2]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VEC-NEXT: br i1 [[TMP11]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; VEC: for.end:
+; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VEC-NEXT: br i1 [[TMP5]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VEC: [[FOR_END]]:
; VEC-NEXT: ret void
;
entry:
@@ -481,84 +490,87 @@ for.end:
}
define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
-; UNROLL-LABEL: @minimal_bit_widths_with_aliasing_store(
-; UNROLL-NEXT: entry:
-; UNROLL-NEXT: br label [[FOR_BODY:%.*]]
-; UNROLL: vector.body:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; UNROLL-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[INDEX]]
-; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP1]]
-; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
+; UNROLL-LABEL: define void @minimal_bit_widths_with_aliasing_store(
+; UNROLL-SAME: i1 [[C:%.*]], ptr [[PTR:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL: [[VECTOR_BODY]]:
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
+; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
+; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
+; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
+; UNROLL-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NEXT: store i8 0, ptr [[TMP4]], align 1
-; UNROLL-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; UNROLL: pred.store.if:
-; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: store i8 [[TMP5]], ptr [[TMP4]], align 1
-; UNROLL-NEXT: br label [[FOR_INC]]
-; UNROLL: pred.store.continue2:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_IF]]:
+; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; UNROLL-NEXT: br i1 [[TMP6]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; UNROLL: for.end:
+; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; UNROLL-NEXT: br i1 [[TMP5]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; UNROLL: [[FOR_END]]:
; UNROLL-NEXT: ret void
;
-; UNROLL-NOSIMPLIFY-LABEL: @minimal_bit_widths_with_aliasing_store(
-; UNROLL-NOSIMPLIFY-NEXT: entry:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: vector.ph:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NOSIMPLIFY: vector.body:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[INDEX]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP1]]
+; UNROLL-NOSIMPLIFY-LABEL: define void @minimal_bit_widths_with_aliasing_store(
+; UNROLL-NOSIMPLIFY-SAME: i1 [[C:%.*]], ptr [[PTR:%.*]]) {
+; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP3]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; UNROLL-NOSIMPLIFY: pred.store.if:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: pred.store.continue:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.if1:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP5]], ptr [[TMP3]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.continue2:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; UNROLL-NOSIMPLIFY: middle.block:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[FOR_INC:%.*]]
-; UNROLL-NOSIMPLIFY: for.end:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[FOR_END:.*]]
+; UNROLL-NOSIMPLIFY: [[FOR_END]]:
; UNROLL-NOSIMPLIFY-NEXT: ret void
;
-; VEC-LABEL: @minimal_bit_widths_with_aliasing_store(
-; VEC-NEXT: entry:
-; VEC-NEXT: br label [[FOR_BODY:%.*]]
-; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[INDEX]]
-; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
-; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP2]], align 1
-; VEC-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.if:
-; VEC-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP5]], ptr [[TMP2]], align 1
-; VEC-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP7]]
-; VEC-NEXT: [[TMP9:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP9]], ptr [[TMP8]], align 1
-; VEC-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.continue2:
+; VEC-LABEL: define void @minimal_bit_widths_with_aliasing_store(
+; VEC-SAME: i1 [[C:%.*]], ptr [[PTR:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
+; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP0]], align 1
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
+; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
+; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]]
+; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
+; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_CONTINUE2]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VEC-NEXT: br i1 [[TMP10]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; VEC: for.end:
+; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VEC-NEXT: br i1 [[TMP5]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VEC: [[FOR_END]]:
; VEC-NEXT: ret void
;
entry:
@@ -589,104 +601,107 @@ for.end:
}
define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
-; UNROLL-LABEL: @sdiv_with_uniform_ops(
-; UNROLL-NEXT: entry:
-; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL: vector.body:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; UNROLL-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
-; UNROLL: pred.store.if:
-; UNROLL-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0:%.*]]
-; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST:%.*]], align 1
+; UNROLL-LABEL: define void @sdiv_with_uniform_ops(
+; UNROLL-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL: [[VECTOR_BODY]]:
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_IF]]:
+; UNROLL-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
+; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; UNROLL-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
-; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; UNROLL: pred.store.continue2:
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NEXT: br i1 [[TMP3]], label [[LOOP_HEADER:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; UNROLL: loop.header:
-; UNROLL-NEXT: [[IV:%.*]] = phi i16 [ [[INC:%.*]], [[LOOP_LATCH:%.*]] ], [ 99, [[PRED_STORE_CONTINUE2]] ]
-; UNROLL-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[LOOP_LATCH]]
-; UNROLL: then:
+; UNROLL-NEXT: br i1 [[TMP3]], label %[[LOOP_HEADER:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; UNROLL: [[LOOP_HEADER]]:
+; UNROLL-NEXT: [[IV:%.*]] = phi i16 [ [[INC:%.*]], %[[LOOP_LATCH:.*]] ], [ 99, %[[PRED_STORE_CONTINUE2]] ]
+; UNROLL-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; UNROLL: [[THEN]]:
; UNROLL-NEXT: [[DIV:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NEXT: store i16 [[DIV]], ptr [[DST]], align 1
-; UNROLL-NEXT: br label [[LOOP_LATCH]]
-; UNROLL: loop.latch:
+; UNROLL-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL: [[LOOP_LATCH]]:
; UNROLL-NEXT: [[INC]] = add i16 [[IV]], 1
; UNROLL-NEXT: [[EC:%.*]] = icmp eq i16 [[INC]], 100
-; UNROLL-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
-; UNROLL: exit:
+; UNROLL-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL: [[EXIT]]:
; UNROLL-NEXT: ret void
;
-; UNROLL-NOSIMPLIFY-LABEL: @sdiv_with_uniform_ops(
-; UNROLL-NOSIMPLIFY-NEXT: entry:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: vector.ph:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NOSIMPLIFY: vector.body:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; UNROLL-NOSIMPLIFY: pred.store.if:
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0:%.*]]
-; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST:%.*]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: pred.store.continue:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.if1:
+; UNROLL-NOSIMPLIFY-LABEL: define void @sdiv_with_uniform_ops(
+; UNROLL-NOSIMPLIFY-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
+; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
+; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: pred.store.continue2:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; UNROLL-NOSIMPLIFY: middle.block:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[SCALAR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: scalar.ph:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP_HEADER:%.*]]
-; UNROLL-NOSIMPLIFY: loop.header:
-; UNROLL-NOSIMPLIFY-NEXT: [[IV:%.*]] = phi i16 [ 99, [[SCALAR_PH]] ], [ [[INC:%.*]], [[LOOP_LATCH:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[LOOP_LATCH]]
-; UNROLL-NOSIMPLIFY: then:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[SCALAR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP_HEADER:.*]]
+; UNROLL-NOSIMPLIFY: [[LOOP_HEADER]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[IV:%.*]] = phi i16 [ 99, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP_LATCH:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; UNROLL-NOSIMPLIFY: [[THEN]]:
; UNROLL-NOSIMPLIFY-NEXT: [[DIV:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: store i16 [[DIV]], ptr [[DST]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP_LATCH]]
-; UNROLL-NOSIMPLIFY: loop.latch:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL-NOSIMPLIFY: [[LOOP_LATCH]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INC]] = add i16 [[IV]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[EC:%.*]] = icmp eq i16 [[INC]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
-; UNROLL-NOSIMPLIFY: exit:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[EXIT]]:
; UNROLL-NOSIMPLIFY-NEXT: ret void
;
-; VEC-LABEL: @sdiv_with_uniform_ops(
-; VEC-NEXT: entry:
-; VEC-NEXT: br label [[VECTOR_BODY:%.*]]
-; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT: br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.if:
-; VEC-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0:%.*]]
-; VEC-NEXT: store i16 [[TMP1]], ptr [[DST:%.*]], align 1
+; VEC-LABEL: define void @sdiv_with_uniform_ops(
+; VEC-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
+; VEC-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; VEC-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
; VEC-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
-; VEC-NEXT: br label [[PRED_STORE_CONTINUE2]]
-; VEC: pred.store.continue2:
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VEC: [[PRED_STORE_CONTINUE2]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; VEC-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; VEC-NEXT: br i1 [[TMP3]], label [[LOOP_HEADER:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; VEC: loop.header:
-; VEC-NEXT: [[IV:%.*]] = phi i16 [ [[INC:%.*]], [[LOOP_LATCH:%.*]] ], [ 99, [[PRED_STORE_CONTINUE2]] ]
-; VEC-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[LOOP_LATCH]]
-; VEC: then:
+; VEC-NEXT: br i1 [[TMP3]], label %[[LOOP_HEADER:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VEC: [[LOOP_HEADER]]:
+; VEC-NEXT: [[IV:%.*]] = phi i16 [ [[INC:%.*]], %[[LOOP_LATCH:.*]] ], [ 99, %[[PRED_STORE_CONTINUE2]] ]
+; VEC-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; VEC: [[THEN]]:
; VEC-NEXT: [[DIV:%.*]] = sdiv i16 10, [[TMP0]]
; VEC-NEXT: store i16 [[DIV]], ptr [[DST]], align 1
-; VEC-NEXT: br label [[LOOP_LATCH]]
-; VEC: loop.latch:
+; VEC-NEXT: br label %[[LOOP_LATCH]]
+; VEC: [[LOOP_LATCH]]:
; VEC-NEXT: [[INC]] = add i16 [[IV]], 1
; VEC-NEXT: [[EC:%.*]] = icmp eq i16 [[INC]], 100
-; VEC-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
-; VEC: exit:
+; VEC-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
entry:
@@ -711,16 +726,17 @@ exit:
}
define void @sinkable_predicated_store(ptr %A, ptr %B) {
-; UNROLL-LABEL: @sinkable_predicated_store(
-; UNROLL-NEXT: entry:
-; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 404
-; UNROLL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 404
+; UNROLL-LABEL: define void @sinkable_predicated_store(
+; UNROLL-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 404
+; UNROLL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 404
; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_BODY:%.*]]
-; UNROLL: vector.body:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY:.*]]
+; UNROLL: [[VECTOR_BODY]]:
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], [ 0, %[[ENTRY]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
@@ -736,44 +752,45 @@ define void @sinkable_predicated_store(ptr %A, ptr %B) {
; UNROLL-NEXT: store i32 [[TMP10]], ptr [[TMP2]], align 4, !alias.scope [[META11]], !noalias [[META8]]
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; UNROLL: scalar.ph:
-; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 100, [[VECTOR_BODY]] ]
-; UNROLL-NEXT: br label [[LOOP:%.*]]
-; UNROLL: loop:
-; UNROLL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; UNROLL-NEXT: br i1 [[TMP11]], label %[[SCALAR_PH]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; UNROLL: [[SCALAR_PH]]:
+; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 100, %[[VECTOR_BODY]] ]
+; UNROLL-NEXT: br label %[[LOOP:.*]]
+; UNROLL: [[LOOP]]:
+; UNROLL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; UNROLL-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
; UNROLL-NEXT: [[GEP_B:%.*]] = getelementptr i32, ptr [[B]], i64 [[IV]]
; UNROLL-NEXT: [[L:%.*]] = load i32, ptr [[GEP_B]], align 4
; UNROLL-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 0
-; UNROLL-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; UNROLL: if.then:
+; UNROLL-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; UNROLL: [[IF_THEN]]:
; UNROLL-NEXT: store i32 0, ptr [[GEP_A]], align 4
-; UNROLL-NEXT: br label [[LOOP_LATCH]]
-; UNROLL: if.else:
+; UNROLL-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL: [[IF_ELSE]]:
; UNROLL-NEXT: store i32 1, ptr [[GEP_A]], align 4
-; UNROLL-NEXT: br label [[LOOP_LATCH]]
-; UNROLL: loop.latch:
+; UNROLL-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL: [[LOOP_LATCH]]:
; UNROLL-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; UNROLL-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; UNROLL-NEXT: br i1 [[EC]], label [[FOR_END:%.*]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
-; UNROLL: for.end:
+; UNROLL-NEXT: br i1 [[EC]], label %[[FOR_END:.*]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; UNROLL: [[FOR_END]]:
; UNROLL-NEXT: ret void
;
-; UNROLL-NOSIMPLIFY-LABEL: @sinkable_predicated_store(
-; UNROLL-NOSIMPLIFY-NEXT: entry:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
-; UNROLL-NOSIMPLIFY: vector.memcheck:
-; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 404
-; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 404
+; UNROLL-NOSIMPLIFY-LABEL: define void @sinkable_predicated_store(
+; UNROLL-NOSIMPLIFY-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_MEMCHECK]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 404
+; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 404
; UNROLL-NOSIMPLIFY-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; UNROLL-NOSIMPLIFY-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; UNROLL-NOSIMPLIFY-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: vector.ph:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NOSIMPLIFY: vector.body:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
@@ -789,77 +806,78 @@ define void @sinkable_predicated_store(ptr %A, ptr %B) {
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP10]], ptr [[TMP2]], align 4, !alias.scope [[META12]], !noalias [[META9]]
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; UNROLL-NOSIMPLIFY: middle.block:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[SCALAR_PH]]
-; UNROLL-NOSIMPLIFY: scalar.ph:
-; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP:%.*]]
-; UNROLL-NOSIMPLIFY: loop:
-; UNROLL-NOSIMPLIFY-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[SCALAR_PH]]
+; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP:.*]]
+; UNROLL-NOSIMPLIFY: [[LOOP]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
; UNROLL-NOSIMPLIFY-NEXT: [[GEP_B:%.*]] = getelementptr i32, ptr [[B]], i64 [[IV]]
; UNROLL-NOSIMPLIFY-NEXT: [[L:%.*]] = load i32, ptr [[GEP_B]], align 4
; UNROLL-NOSIMPLIFY-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 0
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; UNROLL-NOSIMPLIFY: if.then:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; UNROLL-NOSIMPLIFY: [[IF_THEN]]:
; UNROLL-NOSIMPLIFY-NEXT: store i32 0, ptr [[GEP_A]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP_LATCH]]
-; UNROLL-NOSIMPLIFY: if.else:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL-NOSIMPLIFY: [[IF_ELSE]]:
; UNROLL-NOSIMPLIFY-NEXT: store i32 1, ptr [[GEP_A]], align 4
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP_LATCH]]
-; UNROLL-NOSIMPLIFY: loop.latch:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL-NOSIMPLIFY: [[LOOP_LATCH]]:
; UNROLL-NOSIMPLIFY-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[EC]], label [[FOR_END:%.*]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; UNROLL-NOSIMPLIFY: for.end:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[EC]], label %[[FOR_END:.*]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[FOR_END]]:
; UNROLL-NOSIMPLIFY-NEXT: ret void
;
-; VEC-LABEL: @sinkable_predicated_store(
-; VEC-NEXT: entry:
-; VEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 404
-; VEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 404
+; VEC-LABEL: define void @sinkable_predicated_store(
+; VEC-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 404
+; VEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 404
; VEC-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; VEC-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; VEC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; VEC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_BODY:%.*]]
-; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
-; VEC-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP1]]
-; VEC-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META8:![0-9]+]]
-; VEC-NEXT: [[TMP5:%.*]] = icmp ne <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; VEC-NEXT: [[TMP6:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> splat (i32 1), <2 x i32> zeroinitializer
-; VEC-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP6]], i64 0
-; VEC-NEXT: store i32 [[TMP8]], ptr [[TMP2]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
-; VEC-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i64 1
-; VEC-NEXT: store i32 [[TMP7]], ptr [[TMP3]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; VEC-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], [ 0, %[[ENTRY]] ]
+; VEC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; VEC-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
+; VEC-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META8:![0-9]+]]
+; VEC-NEXT: [[TMP4:%.*]] = icmp ne <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; VEC-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> splat (i32 1), <2 x i32> zeroinitializer
+; VEC-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
+; VEC-NEXT: store i32 [[TMP6]], ptr [[TMP1]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
+; VEC-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
+; VEC-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4, !alias.scope [[META11]], !noalias [[META8]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VEC-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; VEC: scalar.ph:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 100, [[VECTOR_BODY]] ]
-; VEC-NEXT: br label [[LOOP:%.*]]
-; VEC: loop:
-; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; VEC-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VEC-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; VEC: [[SCALAR_PH]]:
+; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 100, %[[VECTOR_BODY]] ]
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; VEC-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
; VEC-NEXT: [[GEP_B:%.*]] = getelementptr i32, ptr [[B]], i64 [[IV]]
; VEC-NEXT: [[L:%.*]] = load i32, ptr [[GEP_B]], align 4
; VEC-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 0
-; VEC-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; VEC: if.then:
+; VEC-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; VEC: [[IF_THEN]]:
; VEC-NEXT: store i32 0, ptr [[GEP_A]], align 4
-; VEC-NEXT: br label [[LOOP_LATCH]]
-; VEC: if.else:
+; VEC-NEXT: br label %[[LOOP_LATCH]]
+; VEC: [[IF_ELSE]]:
; VEC-NEXT: store i32 1, ptr [[GEP_A]], align 4
-; VEC-NEXT: br label [[LOOP_LATCH]]
-; VEC: loop.latch:
+; VEC-NEXT: br label %[[LOOP_LATCH]]
+; VEC: [[LOOP_LATCH]]:
; VEC-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; VEC-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; VEC-NEXT: br i1 [[EC]], label [[FOR_END:%.*]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
-; VEC: for.end:
+; VEC-NEXT: br i1 [[EC]], label %[[FOR_END:.*]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; VEC: [[FOR_END]]:
; VEC-NEXT: ret void
;
entry:
@@ -891,12 +909,13 @@ for.end:
}
define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
-; UNROLL-LABEL: @hoistable_predicated_store(
-; UNROLL-NEXT: entry:
-; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 4
-; UNROLL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 4
-; UNROLL-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[D:%.*]], i64 404
-; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
+; UNROLL-LABEL: define void @hoistable_predicated_store(
+; UNROLL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 4
+; UNROLL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 4
+; UNROLL-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[D]], i64 404
+; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 4
; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[C]], [[SCEVGEP1]]
; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -916,39 +935,40 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; UNROLL-NEXT: [[BOUND116:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; UNROLL-NEXT: [[FOUND_CONFLICT17:%.*]] = and i1 [[BOUND015]], [[BOUND116]]
; UNROLL-NEXT: [[CONFLICT_RDX18:%.*]] = or i1 [[CONFLICT_RDX14]], [[FOUND_CONFLICT17]]
-; UNROLL-NEXT: br i1 [[CONFLICT_RDX18]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL: vector.ph:
+; UNROLL-NEXT: br i1 [[CONFLICT_RDX18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 8, !alias.scope [[META15:![0-9]+]]
-; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL: vector.body:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL: [[VECTOR_BODY]]:
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; UNROLL-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
; UNROLL-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META23:![0-9]+]], !noalias [[META24:![0-9]+]]
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
-; UNROLL: scalar.ph:
-; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 100, [[VECTOR_BODY]] ]
-; UNROLL-NEXT: br label [[LOOP:%.*]]
-; UNROLL: loop:
-; UNROLL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; UNROLL-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; UNROLL: [[SCALAR_PH]]:
+; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 100, %[[VECTOR_BODY]] ]
+; UNROLL-NEXT: br label %[[LOOP:.*]]
+; UNROLL: [[LOOP]]:
+; UNROLL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; UNROLL-NEXT: store i32 0, ptr [[C]], align 4
; UNROLL-NEXT: [[L_0:%.*]] = load i32, ptr [[A]], align 8
; UNROLL-NEXT: store i32 [[L_0]], ptr [[B]], align 4
; UNROLL-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; UNROLL-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; UNROLL-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP26:![0-9]+]]
-; UNROLL: exit:
+; UNROLL-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP26:![0-9]+]]
+; UNROLL: [[EXIT]]:
; UNROLL-NEXT: ret void
;
-; UNROLL-NOSIMPLIFY-LABEL: @hoistable_predicated_store(
-; UNROLL-NOSIMPLIFY-NEXT: entry:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
-; UNROLL-NOSIMPLIFY: vector.memcheck:
-; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 4
-; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 4
-; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[D:%.*]], i64 404
-; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
+; UNROLL-NOSIMPLIFY-LABEL: define void @hoistable_predicated_store(
+; UNROLL-NOSIMPLIFY-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]]) {
+; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_MEMCHECK]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 4
+; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 4
+; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[D]], i64 404
+; UNROLL-NOSIMPLIFY-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 4
; UNROLL-NOSIMPLIFY-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[C]], [[SCEVGEP1]]
; UNROLL-NOSIMPLIFY-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; UNROLL-NOSIMPLIFY-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -968,50 +988,51 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; UNROLL-NOSIMPLIFY-NEXT: [[BOUND116:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; UNROLL-NOSIMPLIFY-NEXT: [[FOUND_CONFLICT17:%.*]] = and i1 [[BOUND015]], [[BOUND116]]
; UNROLL-NOSIMPLIFY-NEXT: [[CONFLICT_RDX18:%.*]] = or i1 [[CONFLICT_RDX14]], [[FOUND_CONFLICT17]]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[CONFLICT_RDX18]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; UNROLL-NOSIMPLIFY: vector.ph:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[CONFLICT_RDX18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 8, !alias.scope [[META16:![0-9]+]]
-; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
-; UNROLL-NOSIMPLIFY: vector.body:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
+; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; UNROLL-NOSIMPLIFY-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META21:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META24:![0-9]+]], !noalias [[META25:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
-; UNROLL-NOSIMPLIFY: middle.block:
-; UNROLL-NOSIMPLIFY-NEXT: br label [[SCALAR_PH]]
-; UNROLL-NOSIMPLIFY: scalar.ph:
-; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP:%.*]]
-; UNROLL-NOSIMPLIFY: loop:
-; UNROLL-NOSIMPLIFY-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[SCALAR_PH]]
+; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP:.*]]
+; UNROLL-NOSIMPLIFY: [[LOOP]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: store i32 0, ptr [[C]], align 4
; UNROLL-NOSIMPLIFY-NEXT: [[GEP_D:%.*]] = getelementptr i32, ptr [[D]], i64 [[IV]]
; UNROLL-NOSIMPLIFY-NEXT: [[L:%.*]] = load i32, ptr [[GEP_D]], align 4
; UNROLL-NOSIMPLIFY-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 0
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[IF_THEN4:%.*]], label [[IF_ELSE:%.*]]
-; UNROLL-NOSIMPLIFY: if.then4:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[IF_THEN4:.*]], label %[[IF_ELSE:.*]]
+; UNROLL-NOSIMPLIFY: [[IF_THEN4]]:
; UNROLL-NOSIMPLIFY-NEXT: [[L_0:%.*]] = load i32, ptr [[A]], align 8
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP_LATCH]]
-; UNROLL-NOSIMPLIFY: if.else:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL-NOSIMPLIFY: [[IF_ELSE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[L_1:%.*]] = load i32, ptr [[A]], align 8
-; UNROLL-NOSIMPLIFY-NEXT: br label [[LOOP_LATCH]]
-; UNROLL-NOSIMPLIFY: loop.latch:
-; UNROLL-NOSIMPLIFY-NEXT: [[P:%.*]] = phi i32 [ [[L_0]], [[IF_THEN4]] ], [ [[L_1]], [[IF_ELSE]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[LOOP_LATCH]]
+; UNROLL-NOSIMPLIFY: [[LOOP_LATCH]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[P:%.*]] = phi i32 [ [[L_0]], %[[IF_THEN4]] ], [ [[L_1]], %[[IF_ELSE]] ]
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[P]], ptr [[B]], align 4
; UNROLL-NOSIMPLIFY-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
-; UNROLL-NOSIMPLIFY: exit:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; UNROLL-NOSIMPLIFY: [[EXIT]]:
; UNROLL-NOSIMPLIFY-NEXT: ret void
;
-; VEC-LABEL: @hoistable_predicated_store(
-; VEC-NEXT: entry:
-; VEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 4
-; VEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 4
-; VEC-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[D:%.*]], i64 404
-; VEC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
+; VEC-LABEL: define void @hoistable_predicated_store(
+; VEC-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 4
+; VEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 4
+; VEC-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[D]], i64 404
+; VEC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 4
; VEC-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[C]], [[SCEVGEP1]]
; VEC-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; VEC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -1031,27 +1052,27 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; VEC-NEXT: [[BOUND116:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; VEC-NEXT: [[FOUND_CONFLICT17:%.*]] = and i1 [[BOUND015]], [[BOUND116]]
; VEC-NEXT: [[CONFLICT_RDX18:%.*]] = or i1 [[CONFLICT_RDX14]], [[FOUND_CONFLICT17]]
-; VEC-NEXT: br i1 [[CONFLICT_RDX18]], label [[SCALAR_PH:%.*]], label [[VECTOR_BODY:%.*]]
-; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; VEC-NEXT: br i1 [[CONFLICT_RDX18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], [ 0, %[[ENTRY]] ]
; VEC-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META18:![0-9]+]]
; VEC-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 8, !alias.scope [[META22:![0-9]+]]
; VEC-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META23:![0-9]+]], !noalias [[META24:![0-9]+]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VEC-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
-; VEC: scalar.ph:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 100, [[VECTOR_BODY]] ]
-; VEC-NEXT: br label [[LOOP:%.*]]
-; VEC: loop:
-; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; VEC-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; VEC: [[SCALAR_PH]]:
+; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 100, %[[VECTOR_BODY]] ]
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VEC-NEXT: store i32 0, ptr [[C]], align 4
; VEC-NEXT: [[L_0:%.*]] = load i32, ptr [[A]], align 8
; VEC-NEXT: store i32 [[L_0]], ptr [[B]], align 4
; VEC-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; VEC-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; VEC-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP26:![0-9]+]]
-; VEC: exit:
+; VEC-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP26:![0-9]+]]
+; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list