[llvm] [X86][LoopVectorize] Enable MaximizeBandwidth by default on X86 (PR #229997)
Sumukh J Bharadwaj via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 10 09:13:40 PDT 2026
https://github.com/amd-subharad updated https://github.com/llvm/llvm-project/pull/229997
>From a3a9ebf45c26b0dde285914b3453452ed18849ca Mon Sep 17 00:00:00 2001
From: Sumukh Bharadwaj <Sumukh.Bharadwaj at amd.com>
Date: Thu, 8 Oct 2026 13:18:41 +0530
Subject: [PATCH 1/2] [X86][LoopVectorize] Pre-commit MaxBW VF-selection tests
(NFC)
Add maxbw-cast-cost.ll and maxbw-partial-reduce.ll at the current
pre-enable baseline, so the enable commit that follows shows the VF
diff directly. maxbw-partial-reduce.ll covers the narrow-input partial
reductions (i8 dot product and SAD) requested in review.
---
.../LoopVectorize/X86/maxbw-cast-cost.ll | 353 ++++++++++++++
.../LoopVectorize/X86/maxbw-partial-reduce.ll | 454 ++++++++++++++++++
2 files changed, 807 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/maxbw-cast-cost.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll
diff --git a/llvm/test/Transforms/LoopVectorize/X86/maxbw-cast-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/maxbw-cast-cost.ll
new file mode 100644
index 0000000000000..6ce84dbde40a7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/maxbw-cast-cost.ll
@@ -0,0 +1,353 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -force-vector-interleave=1 \
+; RUN: -mcpu=znver4 -S %s | FileCheck %s --check-prefix=MAXBW
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; i8 x i8 dot product accumulated to i32. Smallest type is i8, widest is
+; i32. With MaxBW enabled, the VF is chosen from the smallest type, so
+; MaxBW should choose VF=64 (512/8) rather than the natural VF=16 (512/32).
+define void @dot_i8(ptr noalias %weights, ptr noalias %input, ptr noalias %output, i64 %n) {
+; MAXBW-LABEL: define void @dot_i8(
+; MAXBW-SAME: ptr noalias [[WEIGHTS:%.*]], ptr noalias [[INPUT:%.*]], ptr noalias [[OUTPUT:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; MAXBW-NEXT: [[ITER_CHECK:.*]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; MAXBW: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; MAXBW: [[VECTOR_PH]]:
+; MAXBW-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
+; MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAXBW: [[VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[WEIGHTS]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[TMP0]], align 1
+; MAXBW-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[INPUT]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <64 x i8>, ptr [[TMP1]], align 1
+; MAXBW-NEXT: [[TMP2:%.*]] = sext <64 x i8> [[WIDE_LOAD]] to <64 x i32>
+; MAXBW-NEXT: [[TMP3:%.*]] = zext <64 x i8> [[WIDE_LOAD2]] to <64 x i32>
+; MAXBW-NEXT: [[TMP4:%.*]] = mul nsw <64 x i32> [[TMP2]], [[TMP3]]
+; MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <16 x i32> @llvm.vector.partial.reduce.add.v16i32.v64i32(<16 x i32> [[VEC_PHI]], <64 x i32> [[TMP4]])
+; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; MAXBW-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; MAXBW: [[MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[PARTIAL_REDUCE]])
+; MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; MAXBW: [[VEC_EPILOG_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; MAXBW-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_PH]]:
+; MAXBW-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[N_MOD_VF3:%.*]] = and i64 [[N]], 7
+; MAXBW-NEXT: [[N_VEC4:%.*]] = sub i64 [[N]], [[N_MOD_VF3]]
+; MAXBW-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; MAXBW-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAXBW: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI6:%.*]] = phi <8 x i32> [ [[TMP17]], %[[VEC_EPILOG_PH]] ], [ [[TMP14:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[WEIGHTS]], i64 [[INDEX5]]
+; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i8>, ptr [[TMP9]], align 1
+; MAXBW-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[INPUT]], i64 [[INDEX5]]
+; MAXBW-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i8>, ptr [[TMP10]], align 1
+; MAXBW-NEXT: [[TMP11:%.*]] = sext <8 x i8> [[WIDE_LOAD7]] to <8 x i32>
+; MAXBW-NEXT: [[TMP12:%.*]] = zext <8 x i8> [[WIDE_LOAD8]] to <8 x i32>
+; MAXBW-NEXT: [[TMP13:%.*]] = mul nsw <8 x i32> [[TMP11]], [[TMP12]]
+; MAXBW-NEXT: [[TMP14]] = add <8 x i32> [[TMP13]], [[VEC_PHI6]]
+; MAXBW-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], 8
+; MAXBW-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC4]]
+; MAXBW-NEXT: br i1 [[TMP15]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP14]])
+; MAXBW-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC4]]
+; MAXBW-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; MAXBW: [[VEC_EPILOG_SCALAR_PH]]:
+; MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: br label %[[LOOP:.*]]
+; MAXBW: [[LOOP]]:
+; MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[GW:%.*]] = getelementptr inbounds i8, ptr [[WEIGHTS]], i64 [[IV]]
+; MAXBW-NEXT: [[W:%.*]] = load i8, ptr [[GW]], align 1
+; MAXBW-NEXT: [[GI:%.*]] = getelementptr inbounds i8, ptr [[INPUT]], i64 [[IV]]
+; MAXBW-NEXT: [[I:%.*]] = load i8, ptr [[GI]], align 1
+; MAXBW-NEXT: [[WE:%.*]] = sext i8 [[W]] to i32
+; MAXBW-NEXT: [[IE:%.*]] = zext i8 [[I]] to i32
+; MAXBW-NEXT: [[MUL:%.*]] = mul nsw i32 [[WE]], [[IE]]
+; MAXBW-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM]]
+; MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; MAXBW-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; MAXBW-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; MAXBW: [[EXIT]]:
+; MAXBW-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; MAXBW-NEXT: store i32 [[ADD_LCSSA]], ptr [[OUTPUT]], align 4
+; MAXBW-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sum = phi i32 [ 0, %entry ], [ %add, %loop ]
+ %gw = getelementptr inbounds i8, ptr %weights, i64 %iv
+ %w = load i8, ptr %gw
+ %gi = getelementptr inbounds i8, ptr %input, i64 %iv
+ %i = load i8, ptr %gi
+ %we = sext i8 %w to i32
+ %ie = zext i8 %i to i32
+ %mul = mul nsw i32 %we, %ie
+ %add = add nsw i32 %mul, %sum
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp eq i64 %iv.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store i32 %add, ptr %output
+ ret void
+}
+
+; Four i8 loads, sext to i32, multiply pairs, add, store i32.
+; Smallest type=i8, widest=i32. MaxBW widens to VF=64.
+define void @multi_sext_dot(ptr noalias %a, ptr noalias %b,
+; MAXBW-LABEL: define void @multi_sext_dot(
+; MAXBW-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], ptr noalias [[D:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; MAXBW-NEXT: [[ITER_CHECK:.*]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; MAXBW: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; MAXBW: [[VECTOR_PH]]:
+; MAXBW-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
+; MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAXBW: [[VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[TMP0]], align 1
+; MAXBW-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <64 x i8>, ptr [[TMP1]], align 1
+; MAXBW-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD3:%.*]] = load <64 x i8>, ptr [[TMP2]], align 1
+; MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[D]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD4:%.*]] = load <64 x i8>, ptr [[TMP3]], align 1
+; MAXBW-NEXT: [[TMP4:%.*]] = sext <64 x i8> [[WIDE_LOAD]] to <64 x i32>
+; MAXBW-NEXT: [[TMP5:%.*]] = sext <64 x i8> [[WIDE_LOAD2]] to <64 x i32>
+; MAXBW-NEXT: [[TMP6:%.*]] = sext <64 x i8> [[WIDE_LOAD3]] to <64 x i32>
+; MAXBW-NEXT: [[TMP7:%.*]] = sext <64 x i8> [[WIDE_LOAD4]] to <64 x i32>
+; MAXBW-NEXT: [[TMP8:%.*]] = mul nsw <64 x i32> [[TMP4]], [[TMP5]]
+; MAXBW-NEXT: [[TMP9:%.*]] = mul nsw <64 x i32> [[TMP6]], [[TMP7]]
+; MAXBW-NEXT: [[TMP10:%.*]] = add nsw <64 x i32> [[TMP8]], [[TMP9]]
+; MAXBW-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; MAXBW-NEXT: store <64 x i32> [[TMP10]], ptr [[TMP11]], align 4
+; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; MAXBW-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; MAXBW: [[MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; MAXBW: [[VEC_EPILOG_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; MAXBW-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; MAXBW: [[VEC_EPILOG_PH]]:
+; MAXBW-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[N_MOD_VF5:%.*]] = and i64 [[N]], 7
+; MAXBW-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; MAXBW-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAXBW: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX7]]
+; MAXBW-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i8>, ptr [[TMP13]], align 1
+; MAXBW-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX7]]
+; MAXBW-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i8>, ptr [[TMP14]], align 1
+; MAXBW-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX7]]
+; MAXBW-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i8>, ptr [[TMP15]], align 1
+; MAXBW-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[D]], i64 [[INDEX7]]
+; MAXBW-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i8>, ptr [[TMP16]], align 1
+; MAXBW-NEXT: [[TMP17:%.*]] = sext <8 x i8> [[WIDE_LOAD8]] to <8 x i32>
+; MAXBW-NEXT: [[TMP18:%.*]] = sext <8 x i8> [[WIDE_LOAD9]] to <8 x i32>
+; MAXBW-NEXT: [[TMP19:%.*]] = sext <8 x i8> [[WIDE_LOAD10]] to <8 x i32>
+; MAXBW-NEXT: [[TMP20:%.*]] = sext <8 x i8> [[WIDE_LOAD11]] to <8 x i32>
+; MAXBW-NEXT: [[TMP21:%.*]] = mul nsw <8 x i32> [[TMP17]], [[TMP18]]
+; MAXBW-NEXT: [[TMP22:%.*]] = mul nsw <8 x i32> [[TMP19]], [[TMP20]]
+; MAXBW-NEXT: [[TMP23:%.*]] = add nsw <8 x i32> [[TMP21]], [[TMP22]]
+; MAXBW-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX7]]
+; MAXBW-NEXT: store <8 x i32> [[TMP23]], ptr [[TMP24]], align 4
+; MAXBW-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX7]], 8
+; MAXBW-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
+; MAXBW-NEXT: br i1 [[TMP25]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; MAXBW-NEXT: br i1 [[CMP_N13]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; MAXBW: [[VEC_EPILOG_SCALAR_PH]]:
+; MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: br label %[[LOOP:.*]]
+; MAXBW: [[LOOP]]:
+; MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[GA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; MAXBW-NEXT: [[LA:%.*]] = load i8, ptr [[GA]], align 1
+; MAXBW-NEXT: [[GB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; MAXBW-NEXT: [[LB:%.*]] = load i8, ptr [[GB]], align 1
+; MAXBW-NEXT: [[GC:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[IV]]
+; MAXBW-NEXT: [[LC:%.*]] = load i8, ptr [[GC]], align 1
+; MAXBW-NEXT: [[GD:%.*]] = getelementptr inbounds i8, ptr [[D]], i64 [[IV]]
+; MAXBW-NEXT: [[LD:%.*]] = load i8, ptr [[GD]], align 1
+; MAXBW-NEXT: [[EA:%.*]] = sext i8 [[LA]] to i32
+; MAXBW-NEXT: [[EB:%.*]] = sext i8 [[LB]] to i32
+; MAXBW-NEXT: [[EC:%.*]] = sext i8 [[LC]] to i32
+; MAXBW-NEXT: [[ED:%.*]] = sext i8 [[LD]] to i32
+; MAXBW-NEXT: [[M1:%.*]] = mul nsw i32 [[EA]], [[EB]]
+; MAXBW-NEXT: [[M2:%.*]] = mul nsw i32 [[EC]], [[ED]]
+; MAXBW-NEXT: [[SUM:%.*]] = add nsw i32 [[M1]], [[M2]]
+; MAXBW-NEXT: [[GO:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[IV]]
+; MAXBW-NEXT: store i32 [[SUM]], ptr [[GO]], align 4
+; MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; MAXBW-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; MAXBW-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; MAXBW: [[EXIT]]:
+; MAXBW-NEXT: ret void
+;
+ ptr noalias %c, ptr noalias %d,
+ ptr noalias %out, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ga = getelementptr inbounds i8, ptr %a, i64 %iv
+ %la = load i8, ptr %ga
+ %gb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %lb = load i8, ptr %gb
+ %gc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %lc = load i8, ptr %gc
+ %gd = getelementptr inbounds i8, ptr %d, i64 %iv
+ %ld = load i8, ptr %gd
+ %ea = sext i8 %la to i32
+ %eb = sext i8 %lb to i32
+ %ec = sext i8 %lc to i32
+ %ed = sext i8 %ld to i32
+ %m1 = mul nsw i32 %ea, %eb
+ %m2 = mul nsw i32 %ec, %ed
+ %sum = add nsw i32 %m1, %m2
+ %go = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %go
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp eq i64 %iv.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; i16 add — smallest = widest = i16. Both MaxBW and default select
+; VF = 512/16 = 32. Control case verifying no regression.
+define void @add_i16(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) {
+; MAXBW-LABEL: define void @add_i16(
+; MAXBW-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; MAXBW-NEXT: [[ITER_CHECK:.*]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; MAXBW: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; MAXBW: [[VECTOR_PH]]:
+; MAXBW-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31
+; MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAXBW: [[VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
+; MAXBW-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
+; MAXBW-NEXT: [[TMP2:%.*]] = add <32 x i16> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[C]], i64 [[INDEX]]
+; MAXBW-NEXT: store <32 x i16> [[TMP2]], ptr [[TMP3]], align 2
+; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; MAXBW-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; MAXBW: [[MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; MAXBW: [[VEC_EPILOG_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; MAXBW-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF10:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_PH]]:
+; MAXBW-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[N_MOD_VF3:%.*]] = and i64 [[N]], 3
+; MAXBW-NEXT: [[N_VEC4:%.*]] = sub i64 [[N]], [[N_MOD_VF3]]
+; MAXBW-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAXBW: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDEX5]]
+; MAXBW-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i16>, ptr [[TMP5]], align 2
+; MAXBW-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[INDEX5]]
+; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i16>, ptr [[TMP6]], align 2
+; MAXBW-NEXT: [[TMP7:%.*]] = add <4 x i16> [[WIDE_LOAD6]], [[WIDE_LOAD7]]
+; MAXBW-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[C]], i64 [[INDEX5]]
+; MAXBW-NEXT: store <4 x i16> [[TMP7]], ptr [[TMP8]], align 2
+; MAXBW-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX5]], 4
+; MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC4]]
+; MAXBW-NEXT: br i1 [[TMP9]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC4]]
+; MAXBW-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; MAXBW: [[VEC_EPILOG_SCALAR_PH]]:
+; MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: br label %[[LOOP:.*]]
+; MAXBW: [[LOOP]]:
+; MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[GA:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; MAXBW-NEXT: [[LA:%.*]] = load i16, ptr [[GA]], align 2
+; MAXBW-NEXT: [[GB:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; MAXBW-NEXT: [[LB:%.*]] = load i16, ptr [[GB]], align 2
+; MAXBW-NEXT: [[ADD:%.*]] = add i16 [[LA]], [[LB]]
+; MAXBW-NEXT: [[GC:%.*]] = getelementptr inbounds i16, ptr [[C]], i64 [[IV]]
+; MAXBW-NEXT: store i16 [[ADD]], ptr [[GC]], align 2
+; MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; MAXBW-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; MAXBW-NEXT: br i1 [[COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
+; MAXBW: [[EXIT]]:
+; MAXBW-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ga = getelementptr inbounds i16, ptr %a, i64 %iv
+ %la = load i16, ptr %ga
+ %gb = getelementptr inbounds i16, ptr %b, i64 %iv
+ %lb = load i16, ptr %gb
+ %add = add i16 %la, %lb
+ %gc = getelementptr inbounds i16, ptr %c, i64 %iv
+ store i16 %add, ptr %gc
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp eq i64 %iv.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret void
+}
+;.
+; MAXBW: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; MAXBW: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; MAXBW: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; MAXBW: [[PROF3]] = !{!"branch_weights", i32 8, i32 56}
+; MAXBW: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; MAXBW: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; MAXBW: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; MAXBW: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; MAXBW: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; MAXBW: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; MAXBW: [[PROF10]] = !{!"branch_weights", i32 4, i32 28}
+; MAXBW: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; MAXBW: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll
new file mode 100644
index 0000000000000..8db0d1bd94367
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll
@@ -0,0 +1,454 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -mcpu=znver4 -S %s | FileCheck %s --check-prefix=MAXBW
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -mcpu=znver4 -vectorizer-maximize-bandwidth=false -S %s | FileCheck %s --check-prefix=OFF
+
+; MaximizeBandwidth sizes the VF from the smallest element type, so these
+; narrow-input partial reductions pack to full register width. On znver4
+; (512-bit) the i8 inputs drive VF=64 and the partial.reduce.add accumulates
+; <64 x i32> into <16 x i32>; with MaxBW off the VF is sized from the i32
+; accumulator instead, capping the loop at VF=16.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; u8 x i8 -> i32 dot product (VNNI vpdpbusd shape).
+define i32 @dot_u8_i8(ptr %a, ptr %b, i32 %n) {
+; MAXBW-LABEL: define i32 @dot_u8_i8(
+; MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; MAXBW-NEXT: [[ENTRY:.*]]:
+; MAXBW-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; MAXBW-NEXT: br i1 [[CMP]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]]
+; MAXBW: [[ITER_CHECK]]:
+; MAXBW-NEXT: [[TC:%.*]] = zext nneg i32 [[N]] to i64
+; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 4
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; MAXBW: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 16
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; MAXBW: [[VECTOR_PH]]:
+; MAXBW-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 15
+; MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[TC]], [[TMP0]]
+; MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAXBW: [[VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; MAXBW-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; MAXBW-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAXBW-NEXT: [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; MAXBW-NEXT: [[TMP5:%.*]] = mul nsw <16 x i32> [[TMP3]], [[TMP4]]
+; MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP5]])
+; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; MAXBW-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; MAXBW: [[MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT_LE:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; MAXBW: [[VEC_EPILOG_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; MAXBW-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_PH]]:
+; MAXBW-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[TMP8:%.*]] = and i64 [[TC]], 3
+; MAXBW-NEXT: [[N_VEC3:%.*]] = sub i64 [[TC]], [[TMP8]]
+; MAXBW-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; MAXBW-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAXBW: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
+; MAXBW-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
+; MAXBW-NEXT: [[TMP11:%.*]] = zext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; MAXBW-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX4]]
+; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP12]], align 1
+; MAXBW-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD7]] to <4 x i32>
+; MAXBW-NEXT: [[TMP14:%.*]] = mul nsw <4 x i32> [[TMP11]], [[TMP13]]
+; MAXBW-NEXT: [[TMP15]] = add <4 x i32> [[VEC_PHI5]], [[TMP14]]
+; MAXBW-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4
+; MAXBW-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
+; MAXBW-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; MAXBW-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TC]], [[N_VEC3]]
+; MAXBW-NEXT: br i1 [[CMP_N9]], label %[[EXIT_LE]], label %[[VEC_EPILOG_SCALAR_PH]]
+; MAXBW: [[VEC_EPILOG_SCALAR_PH]]:
+; MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: br label %[[LOOP:.*]]
+; MAXBW: [[LOOP]]:
+; MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[GEPA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; MAXBW-NEXT: [[LA:%.*]] = load i8, ptr [[GEPA]], align 1
+; MAXBW-NEXT: [[ZA:%.*]] = zext i8 [[LA]] to i32
+; MAXBW-NEXT: [[GEPB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; MAXBW-NEXT: [[LB:%.*]] = load i8, ptr [[GEPB]], align 1
+; MAXBW-NEXT: [[SB:%.*]] = sext i8 [[LB]] to i32
+; MAXBW-NEXT: [[MUL:%.*]] = mul nsw i32 [[ZA]], [[SB]]
+; MAXBW-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[MUL]]
+; MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; MAXBW-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[TC]]
+; MAXBW-NEXT: br i1 [[EC]], label %[[EXIT_LE]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; MAXBW: [[EXIT_LE]]:
+; MAXBW-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; MAXBW-NEXT: br label %[[EXIT]]
+; MAXBW: [[EXIT]]:
+; MAXBW-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[EXIT_LE]] ]
+; MAXBW-NEXT: ret i32 [[RES]]
+;
+; OFF-LABEL: define i32 @dot_u8_i8(
+; OFF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; OFF-NEXT: [[ENTRY:.*]]:
+; OFF-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; OFF-NEXT: br i1 [[CMP]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]]
+; OFF: [[ITER_CHECK]]:
+; OFF-NEXT: [[TC:%.*]] = zext nneg i32 [[N]] to i64
+; OFF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 4
+; OFF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; OFF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; OFF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 16
+; OFF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; OFF: [[VECTOR_PH]]:
+; OFF-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 15
+; OFF-NEXT: [[N_VEC:%.*]] = sub i64 [[TC]], [[TMP0]]
+; OFF-NEXT: br label %[[VECTOR_BODY:.*]]
+; OFF: [[VECTOR_BODY]]:
+; OFF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; OFF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; OFF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; OFF-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; OFF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; OFF-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; OFF-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; OFF-NEXT: [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; OFF-NEXT: [[TMP5:%.*]] = mul nsw <16 x i32> [[TMP3]], [[TMP4]]
+; OFF-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP5]])
+; OFF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; OFF-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; OFF-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; OFF: [[MIDDLE_BLOCK]]:
+; OFF-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; OFF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
+; OFF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LE:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; OFF: [[VEC_EPILOG_ITER_CHECK]]:
+; OFF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; OFF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; OFF: [[VEC_EPILOG_PH]]:
+; OFF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; OFF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; OFF-NEXT: [[TMP8:%.*]] = and i64 [[TC]], 3
+; OFF-NEXT: [[N_VEC3:%.*]] = sub i64 [[TC]], [[TMP8]]
+; OFF-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; OFF-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; OFF: [[VEC_EPILOG_VECTOR_BODY]]:
+; OFF-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; OFF-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; OFF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
+; OFF-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
+; OFF-NEXT: [[TMP11:%.*]] = zext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; OFF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX4]]
+; OFF-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP12]], align 1
+; OFF-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD7]] to <4 x i32>
+; OFF-NEXT: [[TMP14:%.*]] = mul nsw <4 x i32> [[TMP11]], [[TMP13]]
+; OFF-NEXT: [[TMP15]] = add <4 x i32> [[VEC_PHI5]], [[TMP14]]
+; OFF-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4
+; OFF-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
+; OFF-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; OFF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; OFF-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; OFF-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TC]], [[N_VEC3]]
+; OFF-NEXT: br i1 [[CMP_N9]], label %[[EXIT_LE]], label %[[VEC_EPILOG_SCALAR_PH]]
+; OFF: [[VEC_EPILOG_SCALAR_PH]]:
+; OFF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; OFF-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; OFF-NEXT: br label %[[LOOP:.*]]
+; OFF: [[LOOP]]:
+; OFF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; OFF-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; OFF-NEXT: [[GEPA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; OFF-NEXT: [[LA:%.*]] = load i8, ptr [[GEPA]], align 1
+; OFF-NEXT: [[ZA:%.*]] = zext i8 [[LA]] to i32
+; OFF-NEXT: [[GEPB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; OFF-NEXT: [[LB:%.*]] = load i8, ptr [[GEPB]], align 1
+; OFF-NEXT: [[SB:%.*]] = sext i8 [[LB]] to i32
+; OFF-NEXT: [[MUL:%.*]] = mul nsw i32 [[ZA]], [[SB]]
+; OFF-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[MUL]]
+; OFF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; OFF-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[TC]]
+; OFF-NEXT: br i1 [[EC]], label %[[EXIT_LE]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; OFF: [[EXIT_LE]]:
+; OFF-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; OFF-NEXT: br label %[[EXIT]]
+; OFF: [[EXIT]]:
+; OFF-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[EXIT_LE]] ]
+; OFF-NEXT: ret i32 [[RES]]
+;
+entry:
+ %cmp = icmp sgt i32 %n, 0
+ br i1 %cmp, label %ph, label %exit
+
+ph:
+ %tc = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+ %sum = phi i32 [ 0, %ph ], [ %add, %loop ]
+ %gepa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %la = load i8, ptr %gepa
+ %za = zext i8 %la to i32
+ %gepb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %lb = load i8, ptr %gepb
+ %sb = sext i8 %lb to i32
+ %mul = mul nsw i32 %za, %sb
+ %add = add nsw i32 %sum, %mul
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %tc
+ br i1 %ec, label %exit.le, label %loop
+
+exit.le:
+ br label %exit
+exit:
+ %res = phi i32 [ 0, %entry ], [ %add, %exit.le ]
+ ret i32 %res
+}
+
+; Sum of absolute differences over u8 inputs into an i32 accumulator.
+define i32 @sad(ptr %a, ptr %b, i32 %n) {
+; MAXBW-LABEL: define i32 @sad(
+; MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; MAXBW-NEXT: [[ENTRY:.*]]:
+; MAXBW-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; MAXBW-NEXT: br i1 [[CMP]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]]
+; MAXBW: [[ITER_CHECK]]:
+; MAXBW-NEXT: [[TC:%.*]] = zext nneg i32 [[N]] to i64
+; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 4
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; MAXBW: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 16
+; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; MAXBW: [[VECTOR_PH]]:
+; MAXBW-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 15
+; MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[TC]], [[TMP0]]
+; MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAXBW: [[VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; MAXBW-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAXBW-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; MAXBW-NEXT: [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; MAXBW-NEXT: [[TMP5:%.*]] = sub nsw <16 x i32> [[TMP3]], [[TMP4]]
+; MAXBW-NEXT: [[TMP6:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP5]], i1 true)
+; MAXBW-NEXT: [[PARTIAL_REDUCE]] = add <16 x i32> [[VEC_PHI]], [[TMP6]]
+; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; MAXBW: [[MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[PARTIAL_REDUCE]])
+; MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
+; MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT_LE:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; MAXBW: [[VEC_EPILOG_ITER_CHECK]]:
+; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; MAXBW-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; MAXBW: [[VEC_EPILOG_PH]]:
+; MAXBW-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; MAXBW-NEXT: [[TMP11:%.*]] = and i64 [[TC]], 3
+; MAXBW-NEXT: [[N_VEC3:%.*]] = sub i64 [[TC]], [[TMP11]]
+; MAXBW-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; MAXBW-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAXBW: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAXBW-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP12]], %[[VEC_EPILOG_PH]] ], [ [[TMP18:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
+; MAXBW-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
+; MAXBW-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; MAXBW-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX4]]
+; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1
+; MAXBW-NEXT: [[TMP19:%.*]] = zext <4 x i8> [[WIDE_LOAD7]] to <4 x i32>
+; MAXBW-NEXT: [[TMP16:%.*]] = sub nsw <4 x i32> [[TMP14]], [[TMP19]]
+; MAXBW-NEXT: [[TMP17:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP16]], i1 true)
+; MAXBW-NEXT: [[TMP18]] = add <4 x i32> [[VEC_PHI5]], [[TMP17]]
+; MAXBW-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4
+; MAXBW-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
+; MAXBW-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; MAXBW: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAXBW-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
+; MAXBW-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TC]], [[N_VEC3]]
+; MAXBW-NEXT: br i1 [[CMP_N9]], label %[[EXIT_LE]], label %[[VEC_EPILOG_SCALAR_PH]]
+; MAXBW: [[VEC_EPILOG_SCALAR_PH]]:
+; MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; MAXBW-NEXT: br label %[[LOOP:.*]]
+; MAXBW: [[LOOP]]:
+; MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; MAXBW-NEXT: [[GEPA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; MAXBW-NEXT: [[LA:%.*]] = load i8, ptr [[GEPA]], align 1
+; MAXBW-NEXT: [[ZA:%.*]] = zext i8 [[LA]] to i32
+; MAXBW-NEXT: [[GEPB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; MAXBW-NEXT: [[LB:%.*]] = load i8, ptr [[GEPB]], align 1
+; MAXBW-NEXT: [[ZB:%.*]] = zext i8 [[LB]] to i32
+; MAXBW-NEXT: [[SUB:%.*]] = sub nsw i32 [[ZA]], [[ZB]]
+; MAXBW-NEXT: [[ABS:%.*]] = tail call i32 @llvm.abs.i32(i32 [[SUB]], i1 true)
+; MAXBW-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[ABS]]
+; MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; MAXBW-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[TC]]
+; MAXBW-NEXT: br i1 [[EC]], label %[[EXIT_LE]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; MAXBW: [[EXIT_LE]]:
+; MAXBW-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; MAXBW-NEXT: br label %[[EXIT]]
+; MAXBW: [[EXIT]]:
+; MAXBW-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[EXIT_LE]] ]
+; MAXBW-NEXT: ret i32 [[RES]]
+;
+; OFF-LABEL: define i32 @sad(
+; OFF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; OFF-NEXT: [[ENTRY:.*]]:
+; OFF-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; OFF-NEXT: br i1 [[CMP]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]]
+; OFF: [[ITER_CHECK]]:
+; OFF-NEXT: [[TC:%.*]] = zext nneg i32 [[N]] to i64
+; OFF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 4
+; OFF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; OFF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; OFF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 16
+; OFF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; OFF: [[VECTOR_PH]]:
+; OFF-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 15
+; OFF-NEXT: [[N_VEC:%.*]] = sub i64 [[TC]], [[TMP0]]
+; OFF-NEXT: br label %[[VECTOR_BODY:.*]]
+; OFF: [[VECTOR_BODY]]:
+; OFF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; OFF-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; OFF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; OFF-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; OFF-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; OFF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; OFF-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; OFF-NEXT: [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; OFF-NEXT: [[TMP5:%.*]] = sub nsw <16 x i32> [[TMP2]], [[TMP4]]
+; OFF-NEXT: [[TMP6:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP5]], i1 true)
+; OFF-NEXT: [[TMP7]] = add <16 x i32> [[VEC_PHI]], [[TMP6]]
+; OFF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; OFF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; OFF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; OFF: [[MIDDLE_BLOCK]]:
+; OFF-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP7]])
+; OFF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
+; OFF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LE:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; OFF: [[VEC_EPILOG_ITER_CHECK]]:
+; OFF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; OFF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; OFF: [[VEC_EPILOG_PH]]:
+; OFF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; OFF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; OFF-NEXT: [[TMP10:%.*]] = and i64 [[TC]], 3
+; OFF-NEXT: [[N_VEC3:%.*]] = sub i64 [[TC]], [[TMP10]]
+; OFF-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; OFF-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; OFF: [[VEC_EPILOG_VECTOR_BODY]]:
+; OFF-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; OFF-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[TMP18:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; OFF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
+; OFF-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP12]], align 1
+; OFF-NEXT: [[TMP13:%.*]] = zext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; OFF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX4]]
+; OFF-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP14]], align 1
+; OFF-NEXT: [[TMP15:%.*]] = zext <4 x i8> [[WIDE_LOAD7]] to <4 x i32>
+; OFF-NEXT: [[TMP16:%.*]] = sub nsw <4 x i32> [[TMP13]], [[TMP15]]
+; OFF-NEXT: [[TMP17:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP16]], i1 true)
+; OFF-NEXT: [[TMP18]] = add <4 x i32> [[VEC_PHI5]], [[TMP17]]
+; OFF-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4
+; OFF-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
+; OFF-NEXT: br i1 [[TMP19]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; OFF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; OFF-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
+; OFF-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TC]], [[N_VEC3]]
+; OFF-NEXT: br i1 [[CMP_N9]], label %[[EXIT_LE]], label %[[VEC_EPILOG_SCALAR_PH]]
+; OFF: [[VEC_EPILOG_SCALAR_PH]]:
+; OFF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; OFF-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; OFF-NEXT: br label %[[LOOP:.*]]
+; OFF: [[LOOP]]:
+; OFF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; OFF-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; OFF-NEXT: [[GEPA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; OFF-NEXT: [[LA:%.*]] = load i8, ptr [[GEPA]], align 1
+; OFF-NEXT: [[ZA:%.*]] = zext i8 [[LA]] to i32
+; OFF-NEXT: [[GEPB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; OFF-NEXT: [[LB:%.*]] = load i8, ptr [[GEPB]], align 1
+; OFF-NEXT: [[ZB:%.*]] = zext i8 [[LB]] to i32
+; OFF-NEXT: [[SUB:%.*]] = sub nsw i32 [[ZA]], [[ZB]]
+; OFF-NEXT: [[ABS:%.*]] = tail call i32 @llvm.abs.i32(i32 [[SUB]], i1 true)
+; OFF-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[ABS]]
+; OFF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; OFF-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[TC]]
+; OFF-NEXT: br i1 [[EC]], label %[[EXIT_LE]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; OFF: [[EXIT_LE]]:
+; OFF-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[TMP20]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; OFF-NEXT: br label %[[EXIT]]
+; OFF: [[EXIT]]:
+; OFF-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[EXIT_LE]] ]
+; OFF-NEXT: ret i32 [[RES]]
+;
+entry:
+ %cmp = icmp sgt i32 %n, 0
+ br i1 %cmp, label %ph, label %exit
+
+ph:
+ %tc = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+ %sum = phi i32 [ 0, %ph ], [ %add, %loop ]
+ %gepa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %la = load i8, ptr %gepa
+ %za = zext i8 %la to i32
+ %gepb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %lb = load i8, ptr %gepb
+ %zb = zext i8 %lb to i32
+ %sub = sub nsw i32 %za, %zb
+ %abs = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %add = add nsw i32 %sum, %abs
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %tc
+ br i1 %ec, label %exit.le, label %loop
+
+exit.le:
+ br label %exit
+exit:
+ %res = phi i32 [ 0, %entry ], [ %add, %exit.le ]
+ ret i32 %res
+}
+
+declare i32 @llvm.abs.i32(i32, i1)
+;.
+; MAXBW: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; MAXBW: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; MAXBW: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; MAXBW: [[PROF3]] = !{!"branch_weights", i32 4, i32 12}
+; MAXBW: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; MAXBW: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; MAXBW: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; MAXBW: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; MAXBW: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+;.
+; OFF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; OFF: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; OFF: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; OFF: [[PROF3]] = !{!"branch_weights", i32 4, i32 12}
+; OFF: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; OFF: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; OFF: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; OFF: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; OFF: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+;.
>From b48001eeebc77a8082c6b494983ea9da44246a36 Mon Sep 17 00:00:00 2001
From: Sumukh Bharadwaj <Sumukh.Bharadwaj at amd.com>
Date: Thu, 8 Oct 2026 13:18:41 +0530
Subject: [PATCH 2/2] [X86][LoopVectorize] Enable MaximizeBandwidth by default
on X86
Enable the MaximizeBandwidth heuristic by default for X86 fixed-width
vectors via shouldMaximizeVectorBandwidth(), matching the AArch64 Neon
precedent, so the loop vectorizer can select a VF from the smallest
element type in mixed-width loops.
Existing LoopVectorize/X86 and PhaseOrdering/X86 tests whose VF changes
under the new default have their CHECK lines regenerated with MaxBW
enabled, so they validate the new behavior directly rather than being
opted out with -vectorizer-maximize-bandwidth=false. The pre-committed
maxbw-cast-cost.ll and maxbw-partial-reduce.ll are regenerated here, so
their diff shows the VF-from-smallest-type change (the i8 dot product
and SAD loops move from VF=16 to VF=64).
The one mispricing this exposes -- the X86 per-part cost tables
under-counting predicate mask-expansion fanout, which let masked /
gather-scatter loops over-widen -- was addressed separately by the
companion X86 TTI cost-model fix in #201666 (already landed), so the
fix stays in the cost model rather than a generic LV-side floor or a
vectorizer-level workaround.
---
llvm/lib/Target/X86/X86TargetTransformInfo.h | 6 +
.../X86/conditional-scalar-assignment.ll | 109 ++--
.../LoopVectorize/X86/cost-model.ll | 26 +-
.../LoopVectorize/X86/gcc-examples.ll | 16 +-
.../LoopVectorize/X86/induction-costs.ll | 30 +-
.../LoopVectorize/X86/masked_load_store.ll | 282 ++++----
.../LoopVectorize/X86/maxbw-partial-reduce.ll | 100 +--
.../Transforms/LoopVectorize/X86/no_fpmath.ll | 2 +-
.../X86/no_fpmath_with_hotness.ll | 2 +-
.../X86/nondetermisitic-widening-cost.ll | 48 +-
.../Transforms/LoopVectorize/X86/pr47437.ll | 85 ++-
.../LoopVectorize/X86/reduction-crash.ll | 28 +-
.../X86/replicating-load-store-costs.ll | 231 +++----
.../LoopVectorize/X86/strided_load_cost.ll | 124 +++-
.../X86/vector_ptr_load_store.ll | 4 +-
.../X86/vectorization-remarks-loopid-dbg.ll | 2 +-
.../X86/vectorization-remarks.ll | 2 +-
.../PhaseOrdering/X86/pixel-splat.ll | 55 +-
.../X86/preserve-access-group.ll | 28 +-
.../X86/vector-reduction-known-first-value.ll | 603 ++++++++++--------
20 files changed, 1050 insertions(+), 733 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 160a71e1affbb..7194560c6fd49 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -64,6 +64,12 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
unsigned getLoadStoreVecRegBitWidth(unsigned AS) const override;
+ bool shouldMaximizeVectorBandwidth(
+ TargetTransformInfo::RegisterKind K) const override {
+ assert(K != TargetTransformInfo::RGK_Scalar &&
+ "Expected vector register kind");
+ return K == TargetTransformInfo::RGK_FixedWidthVector;
+ }
unsigned getMaxInterleaveFactor(ElementCount VF,
bool HasUnorderedReductions) const override;
InstructionCost getArithmeticInstrCost(
diff --git a/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll b/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
index 4b0b5d1cdb24b..72c3998b822af 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
@@ -387,22 +387,59 @@ define i32 @multi_use_cmp_for_csa_int_select(i64 %N, ptr %data, i32 %a) {
; X86-LABEL: define i32 @multi_use_cmp_for_csa_int_select(
; X86-SAME: i64 [[N:%.*]], ptr [[DATA:%.*]], i32 [[A:%.*]]) {
; X86-NEXT: [[ENTRY:.*]]:
+; X86-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; X86-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; X86: [[VECTOR_PH]]:
+; X86-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; X86-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; X86-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
+; X86-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; X86-NEXT: br label %[[LOOP:.*]]
; X86: [[LOOP]]:
-; X86-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; X86-NEXT: [[DATA_PHI:%.*]] = phi i32 [ -1, %[[ENTRY]] ], [ [[SELECT_DATA:%.*]], %[[LOOP]] ]
-; X86-NEXT: [[IDX_PHI:%.*]] = phi i64 [ -1, %[[ENTRY]] ], [ [[SELECT_IDX:%.*]], %[[LOOP]] ]
+; X86-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; X86-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; X86-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; X86-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
+; X86-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[LOOP]] ]
; X86-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV]]
-; X86-NEXT: [[LD:%.*]] = load i32, ptr [[LD_ADDR]], align 4
+; X86-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[LD_ADDR]], align 4
+; X86-NEXT: [[TMP3:%.*]] = icmp slt <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; X86-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; X86-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; X86-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP4]], <4 x i1> [[TMP1]]
+; X86-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
+; X86-NEXT: [[TMP8]] = select <4 x i1> [[TMP3]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
+; X86-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; X86-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; X86-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; X86-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; X86: [[MIDDLE_BLOCK]]:
+; X86-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP7]], <4 x i1> [[TMP6]], i32 -1)
+; X86-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])
+; X86-NEXT: [[TMP12:%.*]] = icmp ne i64 [[TMP11]], -9223372036854775808
+; X86-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP11]], i64 -1
+; X86-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; X86-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; X86: [[SCALAR_PH]]:
+; X86-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; X86-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; X86-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; X86-NEXT: br label %[[LOOP1:.*]]
+; X86: [[LOOP1]]:
+; X86-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; X86-NEXT: [[DATA_PHI:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT_DATA:%.*]], %[[LOOP1]] ]
+; X86-NEXT: [[IDX_PHI:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[SELECT_IDX:%.*]], %[[LOOP1]] ]
+; X86-NEXT: [[LD_ADDR1:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV1]]
+; X86-NEXT: [[LD:%.*]] = load i32, ptr [[LD_ADDR1]], align 4
; X86-NEXT: [[SELECT_CMP:%.*]] = icmp slt i32 [[A]], [[LD]]
; X86-NEXT: [[SELECT_DATA]] = select i1 [[SELECT_CMP]], i32 [[LD]], i32 [[DATA_PHI]]
-; X86-NEXT: [[SELECT_IDX]] = select i1 [[SELECT_CMP]], i64 [[IV]], i64 [[IDX_PHI]]
-; X86-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; X86-NEXT: [[SELECT_IDX]] = select i1 [[SELECT_CMP]], i64 [[IV1]], i64 [[IDX_PHI]]
+; X86-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; X86-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT:.*]], label %[[LOOP]]
+; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; X86: [[EXIT]]:
-; X86-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP]] ]
-; X86-NEXT: [[SELECT_IDX_LCSSA:%.*]] = phi i64 [ [[SELECT_IDX]], %[[LOOP]] ]
+; X86-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; X86-NEXT: [[SELECT_IDX_LCSSA:%.*]] = phi i64 [ [[SELECT_IDX]], %[[LOOP1]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; X86-NEXT: [[IDX:%.*]] = trunc i64 [[SELECT_IDX_LCSSA]] to i32
; X86-NEXT: [[RES:%.*]] = add i32 [[IDX]], [[SELECT_DATA_LCSSA]]
; X86-NEXT: ret i32 [[RES]]
@@ -410,42 +447,42 @@ define i32 @multi_use_cmp_for_csa_int_select(i64 %N, ptr %data, i32 %a) {
; AVX512-LABEL: define i32 @multi_use_cmp_for_csa_int_select(
; AVX512-SAME: i64 [[N:%.*]], ptr [[DATA:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
; AVX512-NEXT: [[ENTRY:.*]]:
-; AVX512-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; AVX512-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; AVX512: [[VECTOR_PH]]:
-; AVX512-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7
+; AVX512-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; AVX512-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; AVX512-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[A]], i64 0
-; AVX512-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; AVX512-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[A]], i64 0
+; AVX512-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; AVX512-NEXT: br label %[[LOOP:.*]]
; AVX512: [[LOOP]]:
; AVX512-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; AVX512-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
-; AVX512-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
-; AVX512-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[TMP1:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[VEC_PHI1:%.*]] = phi <16 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[LOOP]] ]
; AVX512-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV]]
-; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[LD_ADDR]], align 4
-; AVX512-NEXT: [[TMP2:%.*]] = icmp slt <8 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
-; AVX512-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
-; AVX512-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
-; AVX512-NEXT: [[TMP5]] = select i1 [[TMP4]], <8 x i1> [[TMP3]], <8 x i1> [[TMP0]]
-; AVX512-NEXT: [[TMP6]] = select i1 [[TMP4]], <8 x i32> [[WIDE_LOAD]], <8 x i32> [[VEC_PHI]]
-; AVX512-NEXT: [[TMP7]] = select <8 x i1> [[TMP2]], <8 x i64> [[VEC_IND]], <8 x i64> [[VEC_PHI1]]
-; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; AVX512-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i64> [[VEC_IND]], splat (i64 8)
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[LD_ADDR]], align 4
+; AVX512-NEXT: [[TMP3:%.*]] = icmp slt <16 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; AVX512-NEXT: [[TMP4:%.*]] = freeze <16 x i1> [[TMP3]]
+; AVX512-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP4]])
+; AVX512-NEXT: [[TMP6]] = select i1 [[TMP5]], <16 x i1> [[TMP4]], <16 x i1> [[TMP1]]
+; AVX512-NEXT: [[TMP7]] = select i1 [[TMP5]], <16 x i32> [[WIDE_LOAD]], <16 x i32> [[VEC_PHI]]
+; AVX512-NEXT: [[TMP9]] = select <16 x i1> [[TMP3]], <16 x i64> [[VEC_IND]], <16 x i64> [[VEC_PHI1]]
+; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
+; AVX512-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 16)
; AVX512-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; AVX512-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; AVX512: [[MIDDLE_BLOCK]]:
-; AVX512-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v8i32(<8 x i32> [[TMP6]], <8 x i1> [[TMP5]], i32 -1)
-; AVX512-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> [[TMP7]])
+; AVX512-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v16i32(<16 x i32> [[TMP7]], <16 x i1> [[TMP6]], i32 -1)
+; AVX512-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> [[TMP9]])
; AVX512-NEXT: [[TMP11:%.*]] = icmp ne i64 [[TMP10]], -9223372036854775808
; AVX512-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 -1
; AVX512-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; AVX512-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; AVX512: [[SCALAR_PH]]:
; AVX512-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; AVX512-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; AVX512-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
; AVX512-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
; AVX512-NEXT: br label %[[LOOP1:.*]]
; AVX512: [[LOOP1]]:
@@ -461,7 +498,7 @@ define i32 @multi_use_cmp_for_csa_int_select(i64 %N, ptr %data, i32 %a) {
; AVX512-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; AVX512-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
; AVX512: [[EXIT]]:
-; AVX512-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP1]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; AVX512-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP1]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; AVX512-NEXT: [[SELECT_IDX_LCSSA:%.*]] = phi i64 [ [[SELECT_IDX]], %[[LOOP1]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
; AVX512-NEXT: [[IDX:%.*]] = trunc i64 [[SELECT_IDX_LCSSA]] to i32
; AVX512-NEXT: [[RES:%.*]] = add i32 [[IDX]], [[SELECT_DATA_LCSSA]]
@@ -651,7 +688,7 @@ define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr rea
; X86-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
; X86-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; X86-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; X86-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; X86-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; X86: [[MIDDLE_BLOCK]]:
; X86-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP11]], <4 x i1> [[TMP10]], i32 -1)
; X86-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -676,7 +713,7 @@ define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr rea
; X86-NEXT: [[SELECT_A]] = select i1 [[SELECT_CMP]], i32 [[LD_A]], i32 [[A_PHI]]
; X86-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; X86-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
; X86: [[EXIT]]:
; X86-NEXT: [[SELECT_A_LCSSA:%.*]] = phi i32 [ [[SELECT_A]], %[[LOOP1]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; X86-NEXT: ret i32 [[SELECT_A_LCSSA]]
@@ -792,7 +829,7 @@ define i8 @simple_csa_byte_select(i64 %N, ptr %data, i8 %a) {
; X86-NEXT: [[TMP6]] = select i1 [[TMP4]], <16 x i8> [[WIDE_LOAD]], <16 x i8> [[VEC_PHI]]
; X86-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
; X86-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; X86-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; X86-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
; X86: [[MIDDLE_BLOCK]]:
; X86-NEXT: [[TMP8:%.*]] = call i8 @llvm.experimental.vector.extract.last.active.v16i8(<16 x i8> [[TMP6]], <16 x i1> [[TMP5]], i8 -1)
; X86-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -810,7 +847,7 @@ define i8 @simple_csa_byte_select(i64 %N, ptr %data, i8 %a) {
; X86-NEXT: [[SELECT_DATA]] = select i1 [[SELECT_CMP]], i8 [[LD]], i8 [[DATA_PHI]]
; X86-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; X86-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP11:![0-9]+]]
; X86: [[EXIT]]:
; X86-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i8 [ [[SELECT_DATA]], %[[LOOP1]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; X86-NEXT: ret i8 [[SELECT_DATA_LCSSA]]
@@ -914,7 +951,7 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; X86-NEXT: [[TMP13]] = select i1 [[TMP4]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> [[VEC_PHI1]]
; X86-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; X86-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; X86-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; X86-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; X86: [[MIDDLE_BLOCK]]:
; X86-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP6]], <4 x i1> [[TMP5]], i32 -1)
; X86-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP13]], <4 x i1> [[TMP11]], i32 [[TMP8]])
@@ -933,7 +970,7 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; X86-NEXT: [[SELECT_DATA]] = select i1 [[SELECT_CMP]], i32 [[LD]], i32 [[DATA_PHI]]
; X86-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; X86-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP11:![0-9]+]]
+; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP13:![0-9]+]]
; X86: [[EXIT]]:
; X86-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP1]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
; X86-NEXT: ret i32 [[SELECT_DATA_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 42c8fe3011f7d..74ff70589670b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -454,7 +454,7 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[TMP0]]
; CHECK-NEXT: [[UMIN10:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[A]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[UMIN10]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 24
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 32
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[B]], i64 1)
@@ -485,31 +485,31 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT8]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 31
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 32, i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP20]]
; CHECK-NEXT: [[TMP21:%.*]] = trunc i64 [[N_VEC]] to i32
; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[SRC_2]], align 8, !alias.scope [[META6:![0-9]+]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp ne i64 [[TMP22]], 0
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[TMP23]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TMP23]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP24:%.*]] = trunc i64 [[INDEX]] to i32
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i32 [[TMP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP25]], i64 2
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP26]], align 8, !alias.scope [[META9:![0-9]+]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i64, ptr [[TMP25]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i64>, ptr [[TMP29]], align 8, !alias.scope [[META9:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP27:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP28:%.*]] = and <2 x i1> [[BROADCAST_SPLAT]], [[TMP27]]
-; CHECK-NEXT: [[TMP29:%.*]] = zext <2 x i1> [[TMP28]] to <2 x i8>
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i8> [[TMP29]], i64 1
-; CHECK-NEXT: store i8 [[TMP30]], ptr [[DST]], align 1, !alias.scope [[META12:![0-9]+]], !noalias [[META14:![0-9]+]]
+; CHECK-NEXT: [[TMP35:%.*]] = icmp eq <16 x i64> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = and <16 x i1> [[BROADCAST_SPLAT]], [[TMP35]]
+; CHECK-NEXT: [[TMP33:%.*]] = zext <16 x i1> [[TMP32]] to <16 x i8>
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <16 x i8> [[TMP33]], i64 15
+; CHECK-NEXT: store i8 [[TMP34]], ptr [[DST]], align 1, !alias.scope [[META12:![0-9]+]], !noalias [[META14:![0-9]+]]
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll b/llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll
index 05e855bc338d2..0b5cda3b96321 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll
@@ -44,17 +44,17 @@ define void @example1() {
ret void
}
-; Select VF=4 because sext <8 x i1> to <8 x i32> is expensive.
+; With bandwidth maximization the i16 -> i32 sext widens to VF=8.
;CHECK-LABEL: @example10b(
-;CHECK: load <4 x i16>
-;CHECK: sext <4 x i16>
-;CHECK: store <4 x i32>
+;CHECK: load <8 x i16>
+;CHECK: sext <8 x i16>
+;CHECK: store <8 x i32>
;CHECK: ret void
;UNROLL-LABEL: @example10b(
-;UNROLL: load <4 x i16>
-;UNROLL: load <4 x i16>
-;UNROLL: store <4 x i32>
-;UNROLL: store <4 x i32>
+;UNROLL: load <8 x i16>
+;UNROLL: load <8 x i16>
+;UNROLL: store <8 x i32>
+;UNROLL: store <8 x i32>
;UNROLL: ret void
define void @example10b(ptr noalias nocapture %sa, ptr noalias nocapture %sb, ptr noalias nocapture %sc, ptr noalias nocapture %ia, ptr noalias nocapture %ib, ptr noalias nocapture %ic) {
br label %1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 6a24d265ca73d..7d0521a82850e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -118,23 +118,23 @@ define void @multiple_truncated_ivs_with_wide_uses(i1 %c, ptr %A, ptr %B) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
-; CHECK-NEXT: [[STEP_ADD3:%.*]] = add <4 x i32> [[VEC_IND2]], splat (i32 4)
-; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[C]], <4 x i16> [[VEC_IND]], <4 x i16> splat (i16 10)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C]], <4 x i16> [[STEP_ADD]], <4 x i16> splat (i16 10)
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add <8 x i16> [[VEC_IND]], splat (i16 8)
+; CHECK-NEXT: [[STEP_ADD3:%.*]] = add <8 x i32> [[VEC_IND2]], splat (i32 8)
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[C]], <8 x i16> [[VEC_IND]], <8 x i16> splat (i16 10)
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[C]], <8 x i16> [[STEP_ADD]], <8 x i16> splat (i16 10)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[TMP2]], i64 4
-; CHECK-NEXT: store <4 x i16> [[TMP0]], ptr [[TMP2]], align 2, !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
-; CHECK-NEXT: store <4 x i16> [[TMP1]], ptr [[TMP3]], align 2, !alias.scope [[META6]], !noalias [[META9]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i16, ptr [[TMP2]], i64 8
+; CHECK-NEXT: store <8 x i16> [[TMP8]], ptr [[TMP2]], align 2, !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
+; CHECK-NEXT: store <8 x i16> [[TMP3]], ptr [[TMP5]], align 2, !alias.scope [[META6]], !noalias [[META9]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], i64 4
-; CHECK-NEXT: store <4 x i32> [[VEC_IND2]], ptr [[TMP4]], align 4, !alias.scope [[META9]]
-; CHECK-NEXT: store <4 x i32> [[STEP_ADD3]], ptr [[TMP5]], align 4, !alias.scope [[META9]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], splat (i16 4)
-; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <4 x i32> [[STEP_ADD3]], splat (i32 4)
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP4]], i64 8
+; CHECK-NEXT: store <8 x i32> [[VEC_IND2]], ptr [[TMP4]], align 4, !alias.scope [[META9]]
+; CHECK-NEXT: store <8 x i32> [[STEP_ADD3]], ptr [[TMP7]], align 4, !alias.scope [[META9]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i16> [[STEP_ADD]], splat (i16 8)
+; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <8 x i32> [[STEP_ADD3]], splat (i32 8)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 310d894c159ad..98143a471ad5b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -1532,55 +1532,25 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX512-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 8
; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; AVX512: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; AVX512-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 32
+; AVX512-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 64
; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; AVX512: [[VECTOR_PH]]:
-; AVX512-NEXT: [[N_MOD_VF:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 31
+; AVX512-NEXT: [[N_MOD_VF:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 63
; AVX512-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX512: [[VECTOR_BODY]]:
; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; AVX512-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
-; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
-; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24
-; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; AVX512-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; AVX512-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; AVX512-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP3]], align 1
-; AVX512-NEXT: [[TMP4:%.*]] = and <8 x i8> [[WIDE_LOAD]], splat (i8 1)
-; AVX512-NEXT: [[TMP5:%.*]] = and <8 x i8> [[WIDE_LOAD2]], splat (i8 1)
-; AVX512-NEXT: [[TMP6:%.*]] = and <8 x i8> [[WIDE_LOAD3]], splat (i8 1)
-; AVX512-NEXT: [[TMP7:%.*]] = and <8 x i8> [[WIDE_LOAD4]], splat (i8 1)
-; AVX512-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP4]], zeroinitializer
-; AVX512-NEXT: [[TMP9:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
-; AVX512-NEXT: [[TMP10:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
-; AVX512-NEXT: [[TMP11:%.*]] = icmp ne <8 x i8> [[TMP7]], zeroinitializer
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[TMP0]], align 1
+; AVX512-NEXT: [[TMP2:%.*]] = and <64 x i8> [[WIDE_LOAD]], splat (i8 1)
+; AVX512-NEXT: [[TMP3:%.*]] = icmp ne <64 x i8> [[TMP2]], zeroinitializer
; AVX512-NEXT: [[TMP12:%.*]] = getelementptr ptr, ptr [[IN]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP13:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 8
-; AVX512-NEXT: [[TMP14:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 16
-; AVX512-NEXT: [[TMP15:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 24
-; AVX512-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP12]], <8 x i1> [[TMP8]], <8 x ptr> poison)
-; AVX512-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP13]], <8 x i1> [[TMP9]], <8 x ptr> poison)
-; AVX512-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP14]], <8 x i1> [[TMP10]], <8 x ptr> poison)
-; AVX512-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP15]], <8 x i1> [[TMP11]], <8 x ptr> poison)
-; AVX512-NEXT: [[TMP16:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD]], splat (ptr null)
-; AVX512-NEXT: [[TMP17:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD5]], splat (ptr null)
-; AVX512-NEXT: [[TMP18:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD6]], splat (ptr null)
-; AVX512-NEXT: [[TMP19:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD7]], splat (ptr null)
-; AVX512-NEXT: [[TMP20:%.*]] = select <8 x i1> [[TMP8]], <8 x i1> [[TMP16]], <8 x i1> zeroinitializer
-; AVX512-NEXT: [[TMP21:%.*]] = select <8 x i1> [[TMP9]], <8 x i1> [[TMP17]], <8 x i1> zeroinitializer
-; AVX512-NEXT: [[TMP22:%.*]] = select <8 x i1> [[TMP10]], <8 x i1> [[TMP18]], <8 x i1> zeroinitializer
-; AVX512-NEXT: [[TMP23:%.*]] = select <8 x i1> [[TMP11]], <8 x i1> [[TMP19]], <8 x i1> zeroinitializer
+; AVX512-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x ptr> @llvm.masked.load.v64p0.p0(ptr align 8 [[TMP12]], <64 x i1> [[TMP3]], <64 x ptr> poison)
+; AVX512-NEXT: [[TMP5:%.*]] = icmp ne <64 x ptr> [[WIDE_MASKED_LOAD]], splat (ptr null)
+; AVX512-NEXT: [[TMP6:%.*]] = select <64 x i1> [[TMP3]], <64 x i1> [[TMP5]], <64 x i1> zeroinitializer
; AVX512-NEXT: [[TMP24:%.*]] = getelementptr double, ptr [[OUT]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP25:%.*]] = getelementptr double, ptr [[TMP24]], i64 8
-; AVX512-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[TMP24]], i64 16
-; AVX512-NEXT: [[TMP27:%.*]] = getelementptr double, ptr [[TMP24]], i64 24
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP24]], <8 x i1> [[TMP20]])
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP25]], <8 x i1> [[TMP21]])
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP26]], <8 x i1> [[TMP22]])
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP27]], <8 x i1> [[TMP23]])
-; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; AVX512-NEXT: call void @llvm.masked.store.v64f64.p0(<64 x double> splat (double 5.000000e-01), ptr align 8 [[TMP24]], <64 x i1> [[TMP6]])
+; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; AVX512-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; AVX512-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; AVX512: [[MIDDLE_BLOCK]]:
@@ -1588,7 +1558,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX512-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; AVX512: [[VEC_EPILOG_ITER_CHECK]]:
; AVX512-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; AVX512-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF21]]
+; AVX512-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF45:![0-9]+]]
; AVX512: [[VEC_EPILOG_PH]]:
; AVX512-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; AVX512-NEXT: [[N_MOD_VF8:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 7
@@ -1608,7 +1578,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP35]], <8 x i1> [[TMP34]])
; AVX512-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 8
; AVX512-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]]
-; AVX512-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; AVX512-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
; AVX512: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; AVX512-NEXT: [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
; AVX512-NEXT: br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
@@ -1853,63 +1823,33 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX512-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 8
; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; AVX512: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; AVX512-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 32
+; AVX512-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 64
; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; AVX512: [[VECTOR_PH]]:
-; AVX512-NEXT: [[N_MOD_VF:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 31
+; AVX512-NEXT: [[N_MOD_VF:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 63
; AVX512-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX512: [[VECTOR_BODY]]:
; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; AVX512-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
-; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
-; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24
-; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; AVX512-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; AVX512-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; AVX512-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP3]], align 1
-; AVX512-NEXT: [[TMP4:%.*]] = and <8 x i8> [[WIDE_LOAD]], splat (i8 1)
-; AVX512-NEXT: [[TMP5:%.*]] = and <8 x i8> [[WIDE_LOAD2]], splat (i8 1)
-; AVX512-NEXT: [[TMP6:%.*]] = and <8 x i8> [[WIDE_LOAD3]], splat (i8 1)
-; AVX512-NEXT: [[TMP7:%.*]] = and <8 x i8> [[WIDE_LOAD4]], splat (i8 1)
-; AVX512-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP4]], zeroinitializer
-; AVX512-NEXT: [[TMP9:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
-; AVX512-NEXT: [[TMP10:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
-; AVX512-NEXT: [[TMP11:%.*]] = icmp ne <8 x i8> [[TMP7]], zeroinitializer
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[TMP0]], align 1
+; AVX512-NEXT: [[TMP2:%.*]] = and <64 x i8> [[WIDE_LOAD]], splat (i8 1)
+; AVX512-NEXT: [[TMP3:%.*]] = icmp ne <64 x i8> [[TMP2]], zeroinitializer
; AVX512-NEXT: [[TMP12:%.*]] = getelementptr ptr, ptr [[IN]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP13:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 8
-; AVX512-NEXT: [[TMP14:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 16
-; AVX512-NEXT: [[TMP15:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 24
-; AVX512-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP12]], <8 x i1> [[TMP8]], <8 x ptr> poison)
-; AVX512-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP13]], <8 x i1> [[TMP9]], <8 x ptr> poison)
-; AVX512-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP14]], <8 x i1> [[TMP10]], <8 x ptr> poison)
-; AVX512-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <8 x ptr> @llvm.masked.load.v8p0.p0(ptr align 8 [[TMP15]], <8 x i1> [[TMP11]], <8 x ptr> poison)
-; AVX512-NEXT: [[TMP16:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD]], splat (ptr null)
-; AVX512-NEXT: [[TMP17:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD5]], splat (ptr null)
-; AVX512-NEXT: [[TMP18:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD6]], splat (ptr null)
-; AVX512-NEXT: [[TMP19:%.*]] = icmp ne <8 x ptr> [[WIDE_MASKED_LOAD7]], splat (ptr null)
-; AVX512-NEXT: [[TMP20:%.*]] = select <8 x i1> [[TMP8]], <8 x i1> [[TMP16]], <8 x i1> zeroinitializer
-; AVX512-NEXT: [[TMP21:%.*]] = select <8 x i1> [[TMP9]], <8 x i1> [[TMP17]], <8 x i1> zeroinitializer
-; AVX512-NEXT: [[TMP22:%.*]] = select <8 x i1> [[TMP10]], <8 x i1> [[TMP18]], <8 x i1> zeroinitializer
-; AVX512-NEXT: [[TMP23:%.*]] = select <8 x i1> [[TMP11]], <8 x i1> [[TMP19]], <8 x i1> zeroinitializer
+; AVX512-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x ptr> @llvm.masked.load.v64p0.p0(ptr align 8 [[TMP12]], <64 x i1> [[TMP3]], <64 x ptr> poison)
+; AVX512-NEXT: [[TMP5:%.*]] = icmp ne <64 x ptr> [[WIDE_MASKED_LOAD]], splat (ptr null)
+; AVX512-NEXT: [[TMP6:%.*]] = select <64 x i1> [[TMP3]], <64 x i1> [[TMP5]], <64 x i1> zeroinitializer
; AVX512-NEXT: [[TMP24:%.*]] = getelementptr double, ptr [[OUT]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP25:%.*]] = getelementptr double, ptr [[TMP24]], i64 8
-; AVX512-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[TMP24]], i64 16
-; AVX512-NEXT: [[TMP27:%.*]] = getelementptr double, ptr [[TMP24]], i64 24
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP24]], <8 x i1> [[TMP20]])
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP25]], <8 x i1> [[TMP21]])
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP26]], <8 x i1> [[TMP22]])
-; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP27]], <8 x i1> [[TMP23]])
-; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; AVX512-NEXT: call void @llvm.masked.store.v64f64.p0(<64 x double> splat (double 5.000000e-01), ptr align 8 [[TMP24]], <64 x i1> [[TMP6]])
+; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; AVX512-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX512-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; AVX512-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
; AVX512: [[MIDDLE_BLOCK]]:
; AVX512-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; AVX512-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; AVX512: [[VEC_EPILOG_ITER_CHECK]]:
; AVX512-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; AVX512-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF21]]
+; AVX512-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF45]]
; AVX512: [[VEC_EPILOG_PH]]:
; AVX512-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; AVX512-NEXT: [[N_MOD_VF8:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 7
@@ -1929,7 +1869,7 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> splat (double 5.000000e-01), ptr align 8 [[TMP35]], <8 x i1> [[TMP34]])
; AVX512-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 8
; AVX512-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]]
-; AVX512-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; AVX512-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
; AVX512: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; AVX512-NEXT: [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
; AVX512-NEXT: br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
@@ -2002,10 +1942,10 @@ define i32 @reverse_gather(ptr %p) {
; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX2: [[VECTOR_BODY]]:
; AVX2-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP100:%.*]], %[[VECTOR_BODY]] ]
-; AVX2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP101:%.*]], %[[VECTOR_BODY]] ]
-; AVX2-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP102:%.*]], %[[VECTOR_BODY]] ]
-; AVX2-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP103:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI:%.*]] = phi <8 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP196:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP197:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP198:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI3:%.*]] = phi <8 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP199:%.*]], %[[VECTOR_BODY]] ]
; AVX2-NEXT: [[TMP0:%.*]] = sub i32 100, [[INDEX]]
; AVX2-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], -1
; AVX2-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], -2
@@ -2022,6 +1962,22 @@ define i32 @reverse_gather(ptr %p) {
; AVX2-NEXT: [[TMP13:%.*]] = add i32 [[TMP0]], -13
; AVX2-NEXT: [[TMP14:%.*]] = add i32 [[TMP0]], -14
; AVX2-NEXT: [[TMP15:%.*]] = add i32 [[TMP0]], -15
+; AVX2-NEXT: [[TMP68:%.*]] = add i32 [[TMP0]], -16
+; AVX2-NEXT: [[TMP69:%.*]] = add i32 [[TMP0]], -17
+; AVX2-NEXT: [[TMP70:%.*]] = add i32 [[TMP0]], -18
+; AVX2-NEXT: [[TMP71:%.*]] = add i32 [[TMP0]], -19
+; AVX2-NEXT: [[TMP76:%.*]] = add i32 [[TMP0]], -20
+; AVX2-NEXT: [[TMP77:%.*]] = add i32 [[TMP0]], -21
+; AVX2-NEXT: [[TMP78:%.*]] = add i32 [[TMP0]], -22
+; AVX2-NEXT: [[TMP79:%.*]] = add i32 [[TMP0]], -23
+; AVX2-NEXT: [[TMP96:%.*]] = add i32 [[TMP0]], -24
+; AVX2-NEXT: [[TMP97:%.*]] = add i32 [[TMP0]], -25
+; AVX2-NEXT: [[TMP98:%.*]] = add i32 [[TMP0]], -26
+; AVX2-NEXT: [[TMP99:%.*]] = add i32 [[TMP0]], -27
+; AVX2-NEXT: [[TMP100:%.*]] = add i32 [[TMP0]], -28
+; AVX2-NEXT: [[TMP101:%.*]] = add i32 [[TMP0]], -29
+; AVX2-NEXT: [[TMP102:%.*]] = add i32 [[TMP0]], -30
+; AVX2-NEXT: [[TMP103:%.*]] = add i32 [[TMP0]], -31
; AVX2-NEXT: [[TMP16:%.*]] = zext i32 [[TMP0]] to i64
; AVX2-NEXT: [[TMP17:%.*]] = zext i32 [[TMP1]] to i64
; AVX2-NEXT: [[TMP18:%.*]] = zext i32 [[TMP2]] to i64
@@ -2038,6 +1994,22 @@ define i32 @reverse_gather(ptr %p) {
; AVX2-NEXT: [[TMP29:%.*]] = zext i32 [[TMP13]] to i64
; AVX2-NEXT: [[TMP30:%.*]] = zext i32 [[TMP14]] to i64
; AVX2-NEXT: [[TMP31:%.*]] = zext i32 [[TMP15]] to i64
+; AVX2-NEXT: [[TMP144:%.*]] = zext i32 [[TMP68]] to i64
+; AVX2-NEXT: [[TMP145:%.*]] = zext i32 [[TMP69]] to i64
+; AVX2-NEXT: [[TMP146:%.*]] = zext i32 [[TMP70]] to i64
+; AVX2-NEXT: [[TMP147:%.*]] = zext i32 [[TMP71]] to i64
+; AVX2-NEXT: [[TMP148:%.*]] = zext i32 [[TMP76]] to i64
+; AVX2-NEXT: [[TMP149:%.*]] = zext i32 [[TMP77]] to i64
+; AVX2-NEXT: [[TMP150:%.*]] = zext i32 [[TMP78]] to i64
+; AVX2-NEXT: [[TMP151:%.*]] = zext i32 [[TMP79]] to i64
+; AVX2-NEXT: [[TMP200:%.*]] = zext i32 [[TMP96]] to i64
+; AVX2-NEXT: [[TMP201:%.*]] = zext i32 [[TMP97]] to i64
+; AVX2-NEXT: [[TMP202:%.*]] = zext i32 [[TMP98]] to i64
+; AVX2-NEXT: [[TMP203:%.*]] = zext i32 [[TMP99]] to i64
+; AVX2-NEXT: [[TMP204:%.*]] = zext i32 [[TMP100]] to i64
+; AVX2-NEXT: [[TMP205:%.*]] = zext i32 [[TMP101]] to i64
+; AVX2-NEXT: [[TMP206:%.*]] = zext i32 [[TMP102]] to i64
+; AVX2-NEXT: [[TMP207:%.*]] = zext i32 [[TMP103]] to i64
; AVX2-NEXT: [[TMP32:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP16]]
; AVX2-NEXT: [[TMP33:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP17]]
; AVX2-NEXT: [[TMP34:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP18]]
@@ -2054,6 +2026,22 @@ define i32 @reverse_gather(ptr %p) {
; AVX2-NEXT: [[TMP45:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP29]]
; AVX2-NEXT: [[TMP46:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP30]]
; AVX2-NEXT: [[TMP47:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP31]]
+; AVX2-NEXT: [[TMP208:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP144]]
+; AVX2-NEXT: [[TMP209:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP145]]
+; AVX2-NEXT: [[TMP210:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP146]]
+; AVX2-NEXT: [[TMP211:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP147]]
+; AVX2-NEXT: [[TMP84:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP148]]
+; AVX2-NEXT: [[TMP85:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP149]]
+; AVX2-NEXT: [[TMP86:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP150]]
+; AVX2-NEXT: [[TMP87:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP151]]
+; AVX2-NEXT: [[TMP212:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP200]]
+; AVX2-NEXT: [[TMP213:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP201]]
+; AVX2-NEXT: [[TMP214:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP202]]
+; AVX2-NEXT: [[TMP215:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP203]]
+; AVX2-NEXT: [[TMP92:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP204]]
+; AVX2-NEXT: [[TMP93:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP205]]
+; AVX2-NEXT: [[TMP94:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP206]]
+; AVX2-NEXT: [[TMP95:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP207]]
; AVX2-NEXT: [[TMP48:%.*]] = load ptr, ptr [[TMP32]], align 8
; AVX2-NEXT: [[TMP49:%.*]] = load ptr, ptr [[TMP33]], align 8
; AVX2-NEXT: [[TMP50:%.*]] = load ptr, ptr [[TMP34]], align 8
@@ -2070,59 +2058,107 @@ define i32 @reverse_gather(ptr %p) {
; AVX2-NEXT: [[TMP61:%.*]] = load ptr, ptr [[TMP45]], align 8
; AVX2-NEXT: [[TMP62:%.*]] = load ptr, ptr [[TMP46]], align 8
; AVX2-NEXT: [[TMP63:%.*]] = load ptr, ptr [[TMP47]], align 8
+; AVX2-NEXT: [[TMP216:%.*]] = load ptr, ptr [[TMP208]], align 8
+; AVX2-NEXT: [[TMP217:%.*]] = load ptr, ptr [[TMP209]], align 8
+; AVX2-NEXT: [[TMP218:%.*]] = load ptr, ptr [[TMP210]], align 8
+; AVX2-NEXT: [[TMP219:%.*]] = load ptr, ptr [[TMP211]], align 8
+; AVX2-NEXT: [[TMP220:%.*]] = load ptr, ptr [[TMP84]], align 8
+; AVX2-NEXT: [[TMP221:%.*]] = load ptr, ptr [[TMP85]], align 8
+; AVX2-NEXT: [[TMP222:%.*]] = load ptr, ptr [[TMP86]], align 8
+; AVX2-NEXT: [[TMP223:%.*]] = load ptr, ptr [[TMP87]], align 8
+; AVX2-NEXT: [[TMP224:%.*]] = load ptr, ptr [[TMP212]], align 8
+; AVX2-NEXT: [[TMP225:%.*]] = load ptr, ptr [[TMP213]], align 8
+; AVX2-NEXT: [[TMP226:%.*]] = load ptr, ptr [[TMP214]], align 8
+; AVX2-NEXT: [[TMP227:%.*]] = load ptr, ptr [[TMP215]], align 8
+; AVX2-NEXT: [[TMP228:%.*]] = load ptr, ptr [[TMP92]], align 8
+; AVX2-NEXT: [[TMP229:%.*]] = load ptr, ptr [[TMP93]], align 8
+; AVX2-NEXT: [[TMP230:%.*]] = load ptr, ptr [[TMP94]], align 8
+; AVX2-NEXT: [[TMP231:%.*]] = load ptr, ptr [[TMP95]], align 8
; AVX2-NEXT: [[TMP64:%.*]] = load i32, ptr [[TMP48]], align 8
; AVX2-NEXT: [[TMP65:%.*]] = load i32, ptr [[TMP49]], align 8
; AVX2-NEXT: [[TMP66:%.*]] = load i32, ptr [[TMP50]], align 8
; AVX2-NEXT: [[TMP67:%.*]] = load i32, ptr [[TMP51]], align 8
-; AVX2-NEXT: [[TMP68:%.*]] = insertelement <4 x i32> poison, i32 [[TMP64]], i64 0
-; AVX2-NEXT: [[TMP69:%.*]] = insertelement <4 x i32> [[TMP68]], i32 [[TMP65]], i64 1
-; AVX2-NEXT: [[TMP70:%.*]] = insertelement <4 x i32> [[TMP69]], i32 [[TMP66]], i64 2
-; AVX2-NEXT: [[TMP71:%.*]] = insertelement <4 x i32> [[TMP70]], i32 [[TMP67]], i64 3
; AVX2-NEXT: [[TMP72:%.*]] = load i32, ptr [[TMP52]], align 8
; AVX2-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP53]], align 8
; AVX2-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP54]], align 8
; AVX2-NEXT: [[TMP75:%.*]] = load i32, ptr [[TMP55]], align 8
-; AVX2-NEXT: [[TMP76:%.*]] = insertelement <4 x i32> poison, i32 [[TMP72]], i64 0
-; AVX2-NEXT: [[TMP77:%.*]] = insertelement <4 x i32> [[TMP76]], i32 [[TMP73]], i64 1
-; AVX2-NEXT: [[TMP78:%.*]] = insertelement <4 x i32> [[TMP77]], i32 [[TMP74]], i64 2
-; AVX2-NEXT: [[TMP79:%.*]] = insertelement <4 x i32> [[TMP78]], i32 [[TMP75]], i64 3
+; AVX2-NEXT: [[TMP232:%.*]] = insertelement <8 x i32> poison, i32 [[TMP64]], i64 0
+; AVX2-NEXT: [[TMP137:%.*]] = insertelement <8 x i32> [[TMP232]], i32 [[TMP65]], i64 1
+; AVX2-NEXT: [[TMP138:%.*]] = insertelement <8 x i32> [[TMP137]], i32 [[TMP66]], i64 2
+; AVX2-NEXT: [[TMP139:%.*]] = insertelement <8 x i32> [[TMP138]], i32 [[TMP67]], i64 3
+; AVX2-NEXT: [[TMP140:%.*]] = insertelement <8 x i32> [[TMP139]], i32 [[TMP72]], i64 4
+; AVX2-NEXT: [[TMP141:%.*]] = insertelement <8 x i32> [[TMP140]], i32 [[TMP73]], i64 5
+; AVX2-NEXT: [[TMP142:%.*]] = insertelement <8 x i32> [[TMP141]], i32 [[TMP74]], i64 6
+; AVX2-NEXT: [[TMP143:%.*]] = insertelement <8 x i32> [[TMP142]], i32 [[TMP75]], i64 7
; AVX2-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP56]], align 8
; AVX2-NEXT: [[TMP81:%.*]] = load i32, ptr [[TMP57]], align 8
; AVX2-NEXT: [[TMP82:%.*]] = load i32, ptr [[TMP58]], align 8
; AVX2-NEXT: [[TMP83:%.*]] = load i32, ptr [[TMP59]], align 8
-; AVX2-NEXT: [[TMP84:%.*]] = insertelement <4 x i32> poison, i32 [[TMP80]], i64 0
-; AVX2-NEXT: [[TMP85:%.*]] = insertelement <4 x i32> [[TMP84]], i32 [[TMP81]], i64 1
-; AVX2-NEXT: [[TMP86:%.*]] = insertelement <4 x i32> [[TMP85]], i32 [[TMP82]], i64 2
-; AVX2-NEXT: [[TMP87:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP83]], i64 3
; AVX2-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP60]], align 8
; AVX2-NEXT: [[TMP89:%.*]] = load i32, ptr [[TMP61]], align 8
; AVX2-NEXT: [[TMP90:%.*]] = load i32, ptr [[TMP62]], align 8
; AVX2-NEXT: [[TMP91:%.*]] = load i32, ptr [[TMP63]], align 8
-; AVX2-NEXT: [[TMP92:%.*]] = insertelement <4 x i32> poison, i32 [[TMP88]], i64 0
-; AVX2-NEXT: [[TMP93:%.*]] = insertelement <4 x i32> [[TMP92]], i32 [[TMP89]], i64 1
-; AVX2-NEXT: [[TMP94:%.*]] = insertelement <4 x i32> [[TMP93]], i32 [[TMP90]], i64 2
-; AVX2-NEXT: [[TMP95:%.*]] = insertelement <4 x i32> [[TMP94]], i32 [[TMP91]], i64 3
-; AVX2-NEXT: [[TMP96:%.*]] = icmp ne <4 x i32> [[TMP71]], zeroinitializer
-; AVX2-NEXT: [[TMP97:%.*]] = icmp ne <4 x i32> [[TMP79]], zeroinitializer
-; AVX2-NEXT: [[TMP98:%.*]] = icmp ne <4 x i32> [[TMP87]], zeroinitializer
-; AVX2-NEXT: [[TMP99:%.*]] = icmp ne <4 x i32> [[TMP95]], zeroinitializer
-; AVX2-NEXT: [[TMP100]] = or <4 x i1> [[VEC_PHI]], [[TMP96]]
-; AVX2-NEXT: [[TMP101]] = or <4 x i1> [[VEC_PHI1]], [[TMP97]]
-; AVX2-NEXT: [[TMP102]] = or <4 x i1> [[VEC_PHI2]], [[TMP98]]
-; AVX2-NEXT: [[TMP103]] = or <4 x i1> [[VEC_PHI3]], [[TMP99]]
-; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; AVX2-NEXT: [[TMP152:%.*]] = insertelement <8 x i32> poison, i32 [[TMP80]], i64 0
+; AVX2-NEXT: [[TMP153:%.*]] = insertelement <8 x i32> [[TMP152]], i32 [[TMP81]], i64 1
+; AVX2-NEXT: [[TMP154:%.*]] = insertelement <8 x i32> [[TMP153]], i32 [[TMP82]], i64 2
+; AVX2-NEXT: [[TMP155:%.*]] = insertelement <8 x i32> [[TMP154]], i32 [[TMP83]], i64 3
+; AVX2-NEXT: [[TMP156:%.*]] = insertelement <8 x i32> [[TMP155]], i32 [[TMP88]], i64 4
+; AVX2-NEXT: [[TMP157:%.*]] = insertelement <8 x i32> [[TMP156]], i32 [[TMP89]], i64 5
+; AVX2-NEXT: [[TMP158:%.*]] = insertelement <8 x i32> [[TMP157]], i32 [[TMP90]], i64 6
+; AVX2-NEXT: [[TMP159:%.*]] = insertelement <8 x i32> [[TMP158]], i32 [[TMP91]], i64 7
+; AVX2-NEXT: [[TMP160:%.*]] = load i32, ptr [[TMP216]], align 8
+; AVX2-NEXT: [[TMP161:%.*]] = load i32, ptr [[TMP217]], align 8
+; AVX2-NEXT: [[TMP162:%.*]] = load i32, ptr [[TMP218]], align 8
+; AVX2-NEXT: [[TMP163:%.*]] = load i32, ptr [[TMP219]], align 8
+; AVX2-NEXT: [[TMP164:%.*]] = load i32, ptr [[TMP220]], align 8
+; AVX2-NEXT: [[TMP165:%.*]] = load i32, ptr [[TMP221]], align 8
+; AVX2-NEXT: [[TMP166:%.*]] = load i32, ptr [[TMP222]], align 8
+; AVX2-NEXT: [[TMP167:%.*]] = load i32, ptr [[TMP223]], align 8
+; AVX2-NEXT: [[TMP168:%.*]] = insertelement <8 x i32> poison, i32 [[TMP160]], i64 0
+; AVX2-NEXT: [[TMP169:%.*]] = insertelement <8 x i32> [[TMP168]], i32 [[TMP161]], i64 1
+; AVX2-NEXT: [[TMP170:%.*]] = insertelement <8 x i32> [[TMP169]], i32 [[TMP162]], i64 2
+; AVX2-NEXT: [[TMP171:%.*]] = insertelement <8 x i32> [[TMP170]], i32 [[TMP163]], i64 3
+; AVX2-NEXT: [[TMP172:%.*]] = insertelement <8 x i32> [[TMP171]], i32 [[TMP164]], i64 4
+; AVX2-NEXT: [[TMP173:%.*]] = insertelement <8 x i32> [[TMP172]], i32 [[TMP165]], i64 5
+; AVX2-NEXT: [[TMP174:%.*]] = insertelement <8 x i32> [[TMP173]], i32 [[TMP166]], i64 6
+; AVX2-NEXT: [[TMP175:%.*]] = insertelement <8 x i32> [[TMP174]], i32 [[TMP167]], i64 7
+; AVX2-NEXT: [[TMP176:%.*]] = load i32, ptr [[TMP224]], align 8
+; AVX2-NEXT: [[TMP177:%.*]] = load i32, ptr [[TMP225]], align 8
+; AVX2-NEXT: [[TMP178:%.*]] = load i32, ptr [[TMP226]], align 8
+; AVX2-NEXT: [[TMP179:%.*]] = load i32, ptr [[TMP227]], align 8
+; AVX2-NEXT: [[TMP180:%.*]] = load i32, ptr [[TMP228]], align 8
+; AVX2-NEXT: [[TMP181:%.*]] = load i32, ptr [[TMP229]], align 8
+; AVX2-NEXT: [[TMP182:%.*]] = load i32, ptr [[TMP230]], align 8
+; AVX2-NEXT: [[TMP183:%.*]] = load i32, ptr [[TMP231]], align 8
+; AVX2-NEXT: [[TMP184:%.*]] = insertelement <8 x i32> poison, i32 [[TMP176]], i64 0
+; AVX2-NEXT: [[TMP185:%.*]] = insertelement <8 x i32> [[TMP184]], i32 [[TMP177]], i64 1
+; AVX2-NEXT: [[TMP186:%.*]] = insertelement <8 x i32> [[TMP185]], i32 [[TMP178]], i64 2
+; AVX2-NEXT: [[TMP187:%.*]] = insertelement <8 x i32> [[TMP186]], i32 [[TMP179]], i64 3
+; AVX2-NEXT: [[TMP188:%.*]] = insertelement <8 x i32> [[TMP187]], i32 [[TMP180]], i64 4
+; AVX2-NEXT: [[TMP189:%.*]] = insertelement <8 x i32> [[TMP188]], i32 [[TMP181]], i64 5
+; AVX2-NEXT: [[TMP190:%.*]] = insertelement <8 x i32> [[TMP189]], i32 [[TMP182]], i64 6
+; AVX2-NEXT: [[TMP191:%.*]] = insertelement <8 x i32> [[TMP190]], i32 [[TMP183]], i64 7
+; AVX2-NEXT: [[TMP192:%.*]] = icmp ne <8 x i32> [[TMP143]], zeroinitializer
+; AVX2-NEXT: [[TMP193:%.*]] = icmp ne <8 x i32> [[TMP159]], zeroinitializer
+; AVX2-NEXT: [[TMP194:%.*]] = icmp ne <8 x i32> [[TMP175]], zeroinitializer
+; AVX2-NEXT: [[TMP195:%.*]] = icmp ne <8 x i32> [[TMP191]], zeroinitializer
+; AVX2-NEXT: [[TMP196]] = or <8 x i1> [[VEC_PHI]], [[TMP192]]
+; AVX2-NEXT: [[TMP197]] = or <8 x i1> [[VEC_PHI1]], [[TMP193]]
+; AVX2-NEXT: [[TMP198]] = or <8 x i1> [[VEC_PHI2]], [[TMP194]]
+; AVX2-NEXT: [[TMP199]] = or <8 x i1> [[VEC_PHI3]], [[TMP195]]
+; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
; AVX2-NEXT: [[TMP104:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
; AVX2-NEXT: br i1 [[TMP104]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
; AVX2: [[MIDDLE_BLOCK]]:
-; AVX2-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP101]], [[TMP100]]
-; AVX2-NEXT: [[BIN_RDX4:%.*]] = or <4 x i1> [[TMP102]], [[BIN_RDX]]
-; AVX2-NEXT: [[BIN_RDX5:%.*]] = or <4 x i1> [[TMP103]], [[BIN_RDX4]]
-; AVX2-NEXT: [[TMP105:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX5]])
+; AVX2-NEXT: [[BIN_RDX:%.*]] = or <8 x i1> [[TMP197]], [[TMP196]]
+; AVX2-NEXT: [[BIN_RDX4:%.*]] = or <8 x i1> [[TMP198]], [[BIN_RDX]]
+; AVX2-NEXT: [[BIN_RDX5:%.*]] = or <8 x i1> [[TMP199]], [[BIN_RDX4]]
+; AVX2-NEXT: [[TMP105:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[BIN_RDX5]])
; AVX2-NEXT: [[TMP106:%.*]] = freeze i1 [[TMP105]]
; AVX2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP106]], i32 0, i32 1
; AVX2-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; AVX2: [[VEC_EPILOG_ITER_CHECK]]:
-; AVX2-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF33]]
+; AVX2-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF40:![0-9]+]]
; AVX2: [[VEC_EPILOG_PH]]:
; AVX2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; AVX2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -2161,7 +2197,7 @@ define i32 @reverse_gather(ptr %p) {
; AVX2-NEXT: [[TMP133]] = or <4 x i1> [[VEC_PHI7]], [[TMP132]]
; AVX2-NEXT: [[INDEX_NEXT8]] = add nuw i32 [[INDEX6]], 4
; AVX2-NEXT: [[TMP134:%.*]] = icmp eq i32 [[INDEX_NEXT8]], 100
-; AVX2-NEXT: br i1 [[TMP134]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; AVX2-NEXT: br i1 [[TMP134]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; AVX2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; AVX2-NEXT: [[TMP135:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP133]])
; AVX2-NEXT: [[TMP136:%.*]] = freeze i1 [[TMP135]]
@@ -2212,7 +2248,7 @@ define i32 @reverse_gather(ptr %p) {
; AVX512-NEXT: [[TMP14]] = or <8 x i1> [[VEC_PHI3]], [[TMP10]]
; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
; AVX512-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
-; AVX512-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
+; AVX512-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
; AVX512: [[MIDDLE_BLOCK]]:
; AVX512-NEXT: [[BIN_RDX:%.*]] = or <8 x i1> [[TMP12]], [[TMP11]]
; AVX512-NEXT: [[BIN_RDX13:%.*]] = or <8 x i1> [[TMP13]], [[BIN_RDX]]
@@ -2222,7 +2258,7 @@ define i32 @reverse_gather(ptr %p) {
; AVX512-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP17]], i32 0, i32 1
; AVX512-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; AVX512: [[VEC_EPILOG_ITER_CHECK]]:
-; AVX512-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF51:![0-9]+]]
+; AVX512-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF52:![0-9]+]]
; AVX512: [[VEC_EPILOG_PH]]:
; AVX512-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; AVX512-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -2255,7 +2291,7 @@ define i32 @reverse_gather(ptr %p) {
; AVX512-NEXT: [[TMP36]] = or <4 x i1> [[VEC_PHI16]], [[TMP35]]
; AVX512-NEXT: [[INDEX_NEXT19]] = add nuw i32 [[INDEX15]], 4
; AVX512-NEXT: [[TMP37:%.*]] = icmp eq i32 [[INDEX_NEXT19]], 100
-; AVX512-NEXT: br i1 [[TMP37]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
+; AVX512-NEXT: br i1 [[TMP37]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
; AVX512: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; AVX512-NEXT: [[TMP38:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP36]])
; AVX512-NEXT: [[TMP39:%.*]] = freeze i1 [[TMP38]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll
index 8db0d1bd94367..f6c8a34403700 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/maxbw-partial-reduce.ll
@@ -20,59 +20,59 @@ define i32 @dot_u8_i8(ptr %a, ptr %b, i32 %n) {
; MAXBW-NEXT: br i1 [[CMP]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]]
; MAXBW: [[ITER_CHECK]]:
; MAXBW-NEXT: [[TC:%.*]] = zext nneg i32 [[N]] to i64
-; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 4
+; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 8
; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; MAXBW: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 16
+; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 64
; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; MAXBW: [[VECTOR_PH]]:
-; MAXBW-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 15
+; MAXBW-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 63
; MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[TC]], [[TMP0]]
; MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
; MAXBW: [[VECTOR_BODY]]:
; MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; MAXBW-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[TMP1]], align 1
; MAXBW-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
-; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; MAXBW-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; MAXBW-NEXT: [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; MAXBW-NEXT: [[TMP5:%.*]] = mul nsw <16 x i32> [[TMP3]], [[TMP4]]
-; MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP5]])
-; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <64 x i8>, ptr [[TMP2]], align 1
+; MAXBW-NEXT: [[TMP3:%.*]] = zext <64 x i8> [[WIDE_LOAD]] to <64 x i32>
+; MAXBW-NEXT: [[TMP4:%.*]] = sext <64 x i8> [[WIDE_LOAD2]] to <64 x i32>
+; MAXBW-NEXT: [[TMP5:%.*]] = mul nsw <64 x i32> [[TMP3]], [[TMP4]]
+; MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <16 x i32> @llvm.vector.partial.reduce.add.v16i32.v64i32(<16 x i32> [[VEC_PHI]], <64 x i32> [[TMP5]])
+; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; MAXBW-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; MAXBW-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; MAXBW: [[MIDDLE_BLOCK]]:
-; MAXBW-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; MAXBW-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[PARTIAL_REDUCE]])
; MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
; MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT_LE:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; MAXBW: [[VEC_EPILOG_ITER_CHECK]]:
-; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
; MAXBW-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; MAXBW: [[VEC_EPILOG_PH]]:
; MAXBW-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; MAXBW-NEXT: [[TMP8:%.*]] = and i64 [[TC]], 3
+; MAXBW-NEXT: [[TMP8:%.*]] = and i64 [[TC]], 7
; MAXBW-NEXT: [[N_VEC3:%.*]] = sub i64 [[TC]], [[TMP8]]
-; MAXBW-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; MAXBW-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; MAXBW-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; MAXBW: [[VEC_EPILOG_VECTOR_BODY]]:
; MAXBW-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; MAXBW-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; MAXBW-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; MAXBW-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
-; MAXBW-NEXT: [[TMP11:%.*]] = zext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; MAXBW-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP10]], align 1
+; MAXBW-NEXT: [[TMP11:%.*]] = zext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
; MAXBW-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX4]]
-; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP12]], align 1
-; MAXBW-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD7]] to <4 x i32>
-; MAXBW-NEXT: [[TMP14:%.*]] = mul nsw <4 x i32> [[TMP11]], [[TMP13]]
-; MAXBW-NEXT: [[TMP15]] = add <4 x i32> [[VEC_PHI5]], [[TMP14]]
-; MAXBW-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4
+; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i8>, ptr [[TMP12]], align 1
+; MAXBW-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD7]] to <8 x i32>
+; MAXBW-NEXT: [[TMP14:%.*]] = mul nsw <8 x i32> [[TMP11]], [[TMP13]]
+; MAXBW-NEXT: [[TMP15]] = add <8 x i32> [[VEC_PHI5]], [[TMP14]]
+; MAXBW-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 8
; MAXBW-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
; MAXBW-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; MAXBW: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; MAXBW-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; MAXBW-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP15]])
; MAXBW-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TC]], [[N_VEC3]]
; MAXBW-NEXT: br i1 [[CMP_N9]], label %[[EXIT_LE]], label %[[VEC_EPILOG_SCALAR_PH]]
; MAXBW: [[VEC_EPILOG_SCALAR_PH]]:
@@ -226,28 +226,30 @@ define i32 @sad(ptr %a, ptr %b, i32 %n) {
; MAXBW-NEXT: br i1 [[CMP]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]]
; MAXBW: [[ITER_CHECK]]:
; MAXBW-NEXT: [[TC:%.*]] = zext nneg i32 [[N]] to i64
-; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 4
+; MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 8
; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; MAXBW: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 16
+; MAXBW-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 64
; MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; MAXBW: [[VECTOR_PH]]:
-; MAXBW-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 15
+; MAXBW-NEXT: [[TMP0:%.*]] = and i64 [[TC]], 63
; MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[TC]], [[TMP0]]
; MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
; MAXBW: [[VECTOR_BODY]]:
; MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; MAXBW-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
-; MAXBW-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[TMP1]], align 1
; MAXBW-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
-; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; MAXBW-NEXT: [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; MAXBW-NEXT: [[TMP5:%.*]] = sub nsw <16 x i32> [[TMP3]], [[TMP4]]
-; MAXBW-NEXT: [[TMP6:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP5]], i1 true)
-; MAXBW-NEXT: [[PARTIAL_REDUCE]] = add <16 x i32> [[VEC_PHI]], [[TMP6]]
-; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <64 x i8>, ptr [[TMP2]], align 1
+; MAXBW-NEXT: [[TMP3:%.*]] = freeze <64 x i8> [[WIDE_LOAD]]
+; MAXBW-NEXT: [[TMP4:%.*]] = freeze <64 x i8> [[WIDE_LOAD2]]
+; MAXBW-NEXT: [[TMP5:%.*]] = call <64 x i8> @llvm.umax.v64i8(<64 x i8> [[TMP3]], <64 x i8> [[TMP4]])
+; MAXBW-NEXT: [[TMP6:%.*]] = call <64 x i8> @llvm.umin.v64i8(<64 x i8> [[TMP3]], <64 x i8> [[TMP4]])
+; MAXBW-NEXT: [[TMP7:%.*]] = sub <64 x i8> [[TMP5]], [[TMP6]]
+; MAXBW-NEXT: [[TMP8:%.*]] = zext <64 x i8> [[TMP7]] to <64 x i32>
+; MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <16 x i32> @llvm.vector.partial.reduce.add.v16i32.v64i32(<16 x i32> [[VEC_PHI]], <64 x i32> [[TMP8]])
+; MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; MAXBW: [[MIDDLE_BLOCK]]:
@@ -255,32 +257,32 @@ define i32 @sad(ptr %a, ptr %b, i32 %n) {
; MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
; MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT_LE:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; MAXBW: [[VEC_EPILOG_ITER_CHECK]]:
-; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; MAXBW-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
; MAXBW-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; MAXBW: [[VEC_EPILOG_PH]]:
; MAXBW-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; MAXBW-NEXT: [[TMP11:%.*]] = and i64 [[TC]], 3
+; MAXBW-NEXT: [[TMP11:%.*]] = and i64 [[TC]], 7
; MAXBW-NEXT: [[N_VEC3:%.*]] = sub i64 [[TC]], [[TMP11]]
-; MAXBW-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; MAXBW-NEXT: [[TMP12:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; MAXBW-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; MAXBW: [[VEC_EPILOG_VECTOR_BODY]]:
; MAXBW-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; MAXBW-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP12]], %[[VEC_EPILOG_PH]] ], [ [[TMP18:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; MAXBW-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[TMP12]], %[[VEC_EPILOG_PH]] ], [ [[TMP19:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; MAXBW-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; MAXBW-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
-; MAXBW-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; MAXBW-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP13]], align 1
+; MAXBW-NEXT: [[TMP14:%.*]] = zext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
; MAXBW-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX4]]
-; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1
-; MAXBW-NEXT: [[TMP19:%.*]] = zext <4 x i8> [[WIDE_LOAD7]] to <4 x i32>
-; MAXBW-NEXT: [[TMP16:%.*]] = sub nsw <4 x i32> [[TMP14]], [[TMP19]]
-; MAXBW-NEXT: [[TMP17:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP16]], i1 true)
-; MAXBW-NEXT: [[TMP18]] = add <4 x i32> [[VEC_PHI5]], [[TMP17]]
-; MAXBW-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4
+; MAXBW-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i8>, ptr [[TMP15]], align 1
+; MAXBW-NEXT: [[TMP16:%.*]] = zext <8 x i8> [[WIDE_LOAD7]] to <8 x i32>
+; MAXBW-NEXT: [[TMP17:%.*]] = sub nsw <8 x i32> [[TMP14]], [[TMP16]]
+; MAXBW-NEXT: [[TMP18:%.*]] = call <8 x i32> @llvm.abs.v8i32(<8 x i32> [[TMP17]], i1 true)
+; MAXBW-NEXT: [[TMP19]] = add <8 x i32> [[VEC_PHI5]], [[TMP18]]
+; MAXBW-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 8
; MAXBW-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
; MAXBW-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; MAXBW: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; MAXBW-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
+; MAXBW-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP19]])
; MAXBW-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TC]], [[N_VEC3]]
; MAXBW-NEXT: br i1 [[CMP_N9]], label %[[EXIT_LE]], label %[[VEC_EPILOG_SCALAR_PH]]
; MAXBW: [[VEC_EPILOG_SCALAR_PH]]:
@@ -435,7 +437,7 @@ declare i32 @llvm.abs.i32(i32, i1)
; MAXBW: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; MAXBW: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; MAXBW: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; MAXBW: [[PROF3]] = !{!"branch_weights", i32 4, i32 12}
+; MAXBW: [[PROF3]] = !{!"branch_weights", i32 8, i32 56}
; MAXBW: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
; MAXBW: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
; MAXBW: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll
index ae77b4270ab3e..2056547c2c988 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll
@@ -2,7 +2,7 @@
; CHECK: remark: no_fpmath.c:6:11: loop not vectorized: cannot prove it is safe to reorder floating-point operations
; CHECK: remark: no_fpmath.c:6:14: loop not vectorized
-; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 2, interleaved count: 2)
+; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 4, interleaved count: 2)
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.10.0"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
index cbedbf7fd10c0..6406756f1eead 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
@@ -2,7 +2,7 @@
; CHECK: remark: no_fpmath.c:6:11: loop not vectorized: cannot prove it is safe to reorder floating-point operations (hotness: 300)
; CHECK: remark: no_fpmath.c:6:14: loop not vectorized
-; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 2, interleaved count: 1) (hotness: 300)
+; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 4, interleaved count: 1) (hotness: 300)
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.10.0"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/nondetermisitic-widening-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/nondetermisitic-widening-cost.ll
index 9e473b373faa8..dbd183e18b13d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/nondetermisitic-widening-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/nondetermisitic-widening-cost.ll
@@ -15,47 +15,75 @@ define float @fun(i64 %0, float %1, ptr noalias %a, ptr noalias %b, i64 %len) #
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP0]], 2
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP3]], 1
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], 15
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], 31
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i8], ptr [[VLA]], i64 [[TMP0]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 7
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP23:%.*]] = load ptr, ptr [[TMP15]], align 8
; CHECK-NEXT: [[TMP24:%.*]] = load ptr, ptr [[TMP16]], align 8
; CHECK-NEXT: [[TMP25:%.*]] = load ptr, ptr [[TMP17]], align 8
; CHECK-NEXT: [[TMP26:%.*]] = load ptr, ptr [[TMP18]], align 8
+; CHECK-NEXT: [[TMP28:%.*]] = load ptr, ptr [[TMP20]], align 8
+; CHECK-NEXT: [[TMP29:%.*]] = load ptr, ptr [[TMP21]], align 8
+; CHECK-NEXT: [[TMP30:%.*]] = load ptr, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[TMP27:%.*]] = load ptr, ptr [[TMP19]], align 8
; CHECK-NEXT: [[TMP31:%.*]] = load i64, ptr [[TMP23]], align 8
; CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr [[TMP24]], align 8
; CHECK-NEXT: [[TMP36:%.*]] = load i64, ptr [[TMP25]], align 8
; CHECK-NEXT: [[TMP34:%.*]] = load i64, ptr [[TMP26]], align 8
+; CHECK-NEXT: [[TMP60:%.*]] = load i64, ptr [[TMP28]], align 8
+; CHECK-NEXT: [[TMP33:%.*]] = load i64, ptr [[TMP29]], align 8
+; CHECK-NEXT: [[TMP62:%.*]] = load i64, ptr [[TMP30]], align 8
+; CHECK-NEXT: [[TMP35:%.*]] = load i64, ptr [[TMP27]], align 8
; CHECK-NEXT: [[TMP44:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP31]]
; CHECK-NEXT: [[TMP45:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP32]]
; CHECK-NEXT: [[TMP46:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP36]]
; CHECK-NEXT: [[TMP47:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP64:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP60]]
+; CHECK-NEXT: [[TMP65:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP33]]
+; CHECK-NEXT: [[TMP66:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP62]]
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP35]]
; CHECK-NEXT: [[TMP71:%.*]] = load float, ptr [[TMP44]], align 4
; CHECK-NEXT: [[TMP72:%.*]] = load float, ptr [[TMP45]], align 4
; CHECK-NEXT: [[TMP73:%.*]] = load float, ptr [[TMP46]], align 4
; CHECK-NEXT: [[TMP74:%.*]] = load float, ptr [[TMP47]], align 4
-; CHECK-NEXT: [[TMP75:%.*]] = insertelement <4 x float> poison, float [[TMP71]], i64 0
-; CHECK-NEXT: [[TMP76:%.*]] = insertelement <4 x float> [[TMP75]], float [[TMP72]], i64 1
-; CHECK-NEXT: [[TMP77:%.*]] = insertelement <4 x float> [[TMP76]], float [[TMP73]], i64 2
-; CHECK-NEXT: [[TMP78:%.*]] = insertelement <4 x float> [[TMP77]], float [[TMP74]], i64 3
+; CHECK-NEXT: [[TMP48:%.*]] = load float, ptr [[TMP64]], align 4
+; CHECK-NEXT: [[TMP49:%.*]] = load float, ptr [[TMP65]], align 4
+; CHECK-NEXT: [[TMP50:%.*]] = load float, ptr [[TMP66]], align 4
+; CHECK-NEXT: [[TMP51:%.*]] = load float, ptr [[TMP43]], align 4
+; CHECK-NEXT: [[TMP52:%.*]] = insertelement <8 x float> poison, float [[TMP71]], i64 0
+; CHECK-NEXT: [[TMP53:%.*]] = insertelement <8 x float> [[TMP52]], float [[TMP72]], i64 1
+; CHECK-NEXT: [[TMP54:%.*]] = insertelement <8 x float> [[TMP53]], float [[TMP73]], i64 2
+; CHECK-NEXT: [[TMP55:%.*]] = insertelement <8 x float> [[TMP54]], float [[TMP74]], i64 3
+; CHECK-NEXT: [[TMP56:%.*]] = insertelement <8 x float> [[TMP55]], float [[TMP48]], i64 4
+; CHECK-NEXT: [[TMP57:%.*]] = insertelement <8 x float> [[TMP56]], float [[TMP49]], i64 5
+; CHECK-NEXT: [[TMP58:%.*]] = insertelement <8 x float> [[TMP57]], float [[TMP50]], i64 6
+; CHECK-NEXT: [[TMP59:%.*]] = insertelement <8 x float> [[TMP58]], float [[TMP51]], i64 7
; CHECK-NEXT: [[TMP61:%.*]] = getelementptr [4 x i8], ptr [[VLA]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x float> [[TMP78]], ptr [[TMP61]], align 4
+; CHECK-NEXT: store <8 x float> [[TMP59]], ptr [[TMP61]], align 4
; CHECK-NEXT: [[TMP63:%.*]] = getelementptr [4 x i8], ptr [[TMP4]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x float> [[BROADCAST_SPLAT]], ptr [[TMP63]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: store <8 x float> [[BROADCAST_SPLAT]], ptr [[TMP63]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll b/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
index e546743472b71..533e35d10d73a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
@@ -333,45 +333,82 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; AVX2-NEXT: entry:
; AVX2-NEXT: [[CMP30:%.*]] = icmp sgt i32 [[N:%.*]], 0
; AVX2-NEXT: br i1 [[CMP30]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; AVX2: for.body.preheader:
+; AVX2: iter.check:
; AVX2-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; AVX2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 8
+; AVX2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
; AVX2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AVX2: vector.main.loop.iter.check:
+; AVX2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; AVX2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
; AVX2: vector.ph:
-; AVX2-NEXT: [[N_MOD_VF:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 7
+; AVX2-NEXT: [[N_MOD_VF:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 15
; AVX2-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; AVX2-NEXT: br label [[VECTOR_BODY:%.*]]
; AVX2: vector.body:
-; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; AVX2-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; AVX2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[S1:%.*]], i64 [[TMP1]]
-; AVX2-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP2]], align 2
-; AVX2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; AVX2-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[STRIDED_VEC]] to <8 x i32>
+; AVX2-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP2]], align 2
+; AVX2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; AVX2-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; AVX2-NEXT: [[TMP3:%.*]] = sext <16 x i16> [[STRIDED_VEC]] to <16 x i32>
; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[S2:%.*]], i64 [[TMP1]]
-; AVX2-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i16>, ptr [[TMP5]], align 2
-; AVX2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; AVX2-NEXT: [[TMP7:%.*]] = sext <8 x i16> [[STRIDED_VEC3]] to <8 x i32>
-; AVX2-NEXT: [[TMP8:%.*]] = mul nsw <8 x i32> [[TMP7]], [[TMP4]]
-; AVX2-NEXT: [[TMP9:%.*]] = sext <8 x i16> [[STRIDED_VEC1]] to <8 x i32>
-; AVX2-NEXT: [[TMP10:%.*]] = sext <8 x i16> [[STRIDED_VEC4]] to <8 x i32>
-; AVX2-NEXT: [[TMP11:%.*]] = mul nsw <8 x i32> [[TMP10]], [[TMP9]]
-; AVX2-NEXT: [[TMP12:%.*]] = add nsw <8 x i32> [[TMP11]], [[TMP8]]
+; AVX2-NEXT: [[WIDE_VEC3:%.*]] = load <32 x i16>, ptr [[TMP5]], align 2
+; AVX2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <32 x i16> [[WIDE_VEC3]], <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; AVX2-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <32 x i16> [[WIDE_VEC3]], <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; AVX2-NEXT: [[TMP11:%.*]] = sext <16 x i16> [[STRIDED_VEC4]] to <16 x i32>
+; AVX2-NEXT: [[TMP6:%.*]] = mul nsw <16 x i32> [[TMP11]], [[TMP3]]
+; AVX2-NEXT: [[TMP7:%.*]] = sext <16 x i16> [[STRIDED_VEC2]] to <16 x i32>
+; AVX2-NEXT: [[TMP8:%.*]] = sext <16 x i16> [[STRIDED_VEC5]] to <16 x i32>
+; AVX2-NEXT: [[TMP9:%.*]] = mul nsw <16 x i32> [[TMP8]], [[TMP7]]
+; AVX2-NEXT: [[TMP10:%.*]] = add nsw <16 x i32> [[TMP9]], [[TMP6]]
; AVX2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[D1:%.*]], i64 [[INDEX]]
-; AVX2-NEXT: store <8 x i32> [[TMP12]], ptr [[TMP13]], align 4
-; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; AVX2-NEXT: store <16 x i32> [[TMP10]], ptr [[TMP13]], align 4
+; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; AVX2-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; AVX2-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; AVX2: middle.block:
; AVX2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
-; AVX2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; AVX2: scalar.ph:
-; AVX2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; AVX2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; AVX2: vec.epilog.iter.check:
+; AVX2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; AVX2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; AVX2: vec.epilog.ph:
+; AVX2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; AVX2-NEXT: [[TMP26:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 3
+; AVX2-NEXT: [[N_VEC6:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[TMP26]]
; AVX2-NEXT: br label [[FOR_BODY:%.*]]
+; AVX2: vec.epilog.vector.body:
+; AVX2-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[FOR_BODY]] ]
+; AVX2-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[INDEX7]], 1
+; AVX2-NEXT: [[TMP27:%.*]] = getelementptr inbounds i16, ptr [[S1]], i64 [[TMP14]]
+; AVX2-NEXT: [[WIDE_VEC8:%.*]] = load <8 x i16>, ptr [[TMP27]], align 2
+; AVX2-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x i16> [[WIDE_VEC8]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; AVX2-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x i16> [[WIDE_VEC8]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX2-NEXT: [[TMP28:%.*]] = sext <4 x i16> [[STRIDED_VEC9]] to <4 x i32>
+; AVX2-NEXT: [[TMP29:%.*]] = getelementptr inbounds i16, ptr [[S2]], i64 [[TMP14]]
+; AVX2-NEXT: [[WIDE_VEC11:%.*]] = load <8 x i16>, ptr [[TMP29]], align 2
+; AVX2-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <8 x i16> [[WIDE_VEC11]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; AVX2-NEXT: [[STRIDED_VEC13:%.*]] = shufflevector <8 x i16> [[WIDE_VEC11]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX2-NEXT: [[TMP30:%.*]] = sext <4 x i16> [[STRIDED_VEC12]] to <4 x i32>
+; AVX2-NEXT: [[TMP31:%.*]] = mul nsw <4 x i32> [[TMP30]], [[TMP28]]
+; AVX2-NEXT: [[TMP32:%.*]] = sext <4 x i16> [[STRIDED_VEC10]] to <4 x i32>
+; AVX2-NEXT: [[TMP33:%.*]] = sext <4 x i16> [[STRIDED_VEC13]] to <4 x i32>
+; AVX2-NEXT: [[TMP22:%.*]] = mul nsw <4 x i32> [[TMP33]], [[TMP32]]
+; AVX2-NEXT: [[TMP23:%.*]] = add nsw <4 x i32> [[TMP22]], [[TMP31]]
+; AVX2-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[D1]], i64 [[INDEX7]]
+; AVX2-NEXT: store <4 x i32> [[TMP23]], ptr [[TMP24]], align 4
+; AVX2-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX7]], 4
+; AVX2-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC6]]
+; AVX2-NEXT: br i1 [[TMP25]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AVX2: vec.epilog.middle.block:
+; AVX2-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC6]]
+; AVX2-NEXT: br i1 [[CMP_N15]], label [[FOR_END_LOOPEXIT]], label [[SCALAR_PH]]
+; AVX2: vec.epilog.scalar.ph:
+; AVX2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; AVX2-NEXT: br label [[FOR_BODY1:%.*]]
; AVX2: for.body:
-; AVX2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; AVX2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY1]] ]
; AVX2-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
; AVX2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[S1]], i64 [[TMP16]]
; AVX2-NEXT: [[TMP17:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
@@ -393,7 +430,7 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; AVX2-NEXT: store i32 [[ADD18]], ptr [[ARRAYIDX20]], align 4
; AVX2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; AVX2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; AVX2-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; AVX2-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; AVX2: for.end.loopexit:
; AVX2-NEXT: br label [[FOR_END]]
; AVX2: for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
index 65759d545e4af..dd800f71a819e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
@@ -12,7 +12,7 @@ define void @pr15344(ptr noalias %ar, ptr noalias %ar2, i32 %exit.limit, i1 %con
; CHECK: [[PH]]:
; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK: [[LOOP_PREHEADER]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[EXIT_LIMIT]], 10
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[EXIT_LIMIT]], 12
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = shl i32 [[EXIT_LIMIT]], 2
@@ -24,29 +24,29 @@ define void @pr15344(ptr noalias %ar, ptr noalias %ar2, i32 %exit.limit, i1 %con
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[EXIT_LIMIT]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[EXIT_LIMIT]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[EXIT_LIMIT]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2]] = fadd fast <2 x double> [[VEC_PHI]], splat (double 1.000000e+00)
-; CHECK-NEXT: [[TMP3]] = fadd fast <2 x double> [[VEC_PHI2]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5]] = fadd fast <4 x double> [[VEC_PHI]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[TMP6]] = fadd fast <4 x double> [[VEC_PHI2]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[AR2]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i32 2
-; CHECK-NEXT: store <2 x float> splat (float 2.000000e+00), ptr [[TMP4]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT: store <2 x float> splat (float 2.000000e+00), ptr [[TMP6]], align 4, !alias.scope [[META0]], !noalias [[META3]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i32 4
+; CHECK-NEXT: store <4 x float> splat (float 2.000000e+00), ptr [[TMP4]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: store <4 x float> splat (float 2.000000e+00), ptr [[TMP8]], align 4, !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP3]], [[TMP2]]
-; CHECK-NEXT: [[TMP8:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x double> [[TMP6]], [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[BIN_RDX]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[EXIT_LIMIT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ], [ 0.000000e+00, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ], [ 0.000000e+00, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -61,7 +61,7 @@ define void @pr15344(ptr noalias %ar, ptr noalias %ar2, i32 %exit.limit, i1 %con
; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i32 [[IV_NEXT]], [[EXIT_LIMIT]]
; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
-; CHECK-NEXT: [[FADD_LCSSA:%.*]] = phi double [ [[FADD]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[FADD_LCSSA:%.*]] = phi double [ [[FADD]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RET:%.*]] = phi double [ 0.000000e+00, %[[PH]] ], [ [[FADD_LCSSA]], %[[EXIT_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
index bb005753856ae..83b1d81fcd7df 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
@@ -165,23 +165,17 @@ define void @test_store_initially_interleave(i32 %n, ptr noalias %src) #0 {
; I32-SAME: i32 [[N:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
; I32-NEXT: [[ITER_CHECK:.*:]]
; I32-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
-; I32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 4
-; I32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; I32: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; I32-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i32 [[TMP0]], 16
+; I32-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i32 [[TMP0]], 8
; I32-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; I32: [[VECTOR_PH]]:
-; I32-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP0]], 15
+; I32-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP0]], 7
; I32-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
-; I32-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 16, i32 [[N_MOD_VF]]
+; I32-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 8, i32 [[N_MOD_VF]]
; I32-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]
; I32-NEXT: br label %[[VECTOR_BODY:.*]]
; I32: [[VECTOR_BODY]]:
; I32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; I32-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; I32-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
-; I32-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i32> [[STEP_ADD]], splat (i32 4)
-; I32-NEXT: [[STEP_ADD_3:%.*]] = add nuw <4 x i32> [[STEP_ADD_2]], splat (i32 4)
+; I32-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; I32-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 1
; I32-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 2
; I32-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 3
@@ -189,131 +183,46 @@ define void @test_store_initially_interleave(i32 %n, ptr noalias %src) #0 {
; I32-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 5
; I32-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 6
; I32-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], 7
-; I32-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], 8
-; I32-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], 9
-; I32-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], 10
-; I32-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 11
-; I32-NEXT: [[TMP14:%.*]] = add i32 [[INDEX]], 12
-; I32-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], 13
-; I32-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], 14
-; I32-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 15
-; I32-NEXT: [[TMP18:%.*]] = uitofp <4 x i32> [[VEC_IND]] to <4 x double>
-; I32-NEXT: [[TMP23:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
-; I32-NEXT: [[TMP28:%.*]] = uitofp <4 x i32> [[STEP_ADD_2]] to <4 x double>
-; I32-NEXT: [[TMP33:%.*]] = uitofp <4 x i32> [[STEP_ADD_3]] to <4 x double>
-; I32-NEXT: [[TMP53:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX]]
-; I32-NEXT: [[TMP54:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP3]]
-; I32-NEXT: [[TMP55:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP4]]
-; I32-NEXT: [[TMP56:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP5]]
-; I32-NEXT: [[TMP57:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP6]]
-; I32-NEXT: [[TMP58:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP7]]
-; I32-NEXT: [[TMP59:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP8]]
-; I32-NEXT: [[TMP60:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP9]]
-; I32-NEXT: [[TMP61:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP10]]
-; I32-NEXT: [[TMP62:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP11]]
-; I32-NEXT: [[TMP63:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP12]]
-; I32-NEXT: [[TMP64:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP13]]
-; I32-NEXT: [[TMP65:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP14]]
-; I32-NEXT: [[TMP66:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP15]]
-; I32-NEXT: [[TMP67:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP16]]
-; I32-NEXT: [[TMP68:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP17]]
-; I32-NEXT: [[TMP38:%.*]] = load ptr, ptr [[TMP53]], align 4
-; I32-NEXT: [[TMP39:%.*]] = load ptr, ptr [[TMP54]], align 4
-; I32-NEXT: [[TMP40:%.*]] = load ptr, ptr [[TMP55]], align 4
-; I32-NEXT: [[TMP41:%.*]] = load ptr, ptr [[TMP56]], align 4
-; I32-NEXT: [[TMP42:%.*]] = load ptr, ptr [[TMP57]], align 4
-; I32-NEXT: [[TMP43:%.*]] = load ptr, ptr [[TMP58]], align 4
-; I32-NEXT: [[TMP44:%.*]] = load ptr, ptr [[TMP59]], align 4
-; I32-NEXT: [[TMP45:%.*]] = load ptr, ptr [[TMP60]], align 4
-; I32-NEXT: [[TMP46:%.*]] = load ptr, ptr [[TMP61]], align 4
-; I32-NEXT: [[TMP47:%.*]] = load ptr, ptr [[TMP62]], align 4
-; I32-NEXT: [[TMP48:%.*]] = load ptr, ptr [[TMP63]], align 4
-; I32-NEXT: [[TMP49:%.*]] = load ptr, ptr [[TMP64]], align 4
+; I32-NEXT: [[TMP11:%.*]] = uitofp <8 x i32> [[VEC_IND]] to <8 x double>
+; I32-NEXT: [[TMP65:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX]]
+; I32-NEXT: [[TMP66:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP3]]
+; I32-NEXT: [[TMP67:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP4]]
+; I32-NEXT: [[TMP68:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP5]]
+; I32-NEXT: [[TMP84:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP6]]
+; I32-NEXT: [[TMP85:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP7]]
+; I32-NEXT: [[TMP86:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP8]]
+; I32-NEXT: [[TMP87:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP9]]
; I32-NEXT: [[TMP50:%.*]] = load ptr, ptr [[TMP65]], align 4
; I32-NEXT: [[TMP51:%.*]] = load ptr, ptr [[TMP66]], align 4
; I32-NEXT: [[TMP52:%.*]] = load ptr, ptr [[TMP67]], align 4
; I32-NEXT: [[TMP69:%.*]] = load ptr, ptr [[TMP68]], align 4
-; I32-NEXT: [[TMP19:%.*]] = extractelement <4 x double> [[TMP18]], i64 0
-; I32-NEXT: store double [[TMP19]], ptr [[TMP38]], align 4
-; I32-NEXT: [[TMP20:%.*]] = extractelement <4 x double> [[TMP18]], i64 1
-; I32-NEXT: store double [[TMP20]], ptr [[TMP39]], align 4
-; I32-NEXT: [[TMP21:%.*]] = extractelement <4 x double> [[TMP18]], i64 2
-; I32-NEXT: store double [[TMP21]], ptr [[TMP40]], align 4
-; I32-NEXT: [[TMP22:%.*]] = extractelement <4 x double> [[TMP18]], i64 3
-; I32-NEXT: store double [[TMP22]], ptr [[TMP41]], align 4
-; I32-NEXT: [[TMP24:%.*]] = extractelement <4 x double> [[TMP23]], i64 0
-; I32-NEXT: store double [[TMP24]], ptr [[TMP42]], align 4
-; I32-NEXT: [[TMP25:%.*]] = extractelement <4 x double> [[TMP23]], i64 1
-; I32-NEXT: store double [[TMP25]], ptr [[TMP43]], align 4
-; I32-NEXT: [[TMP26:%.*]] = extractelement <4 x double> [[TMP23]], i64 2
-; I32-NEXT: store double [[TMP26]], ptr [[TMP44]], align 4
-; I32-NEXT: [[TMP27:%.*]] = extractelement <4 x double> [[TMP23]], i64 3
-; I32-NEXT: store double [[TMP27]], ptr [[TMP45]], align 4
-; I32-NEXT: [[TMP29:%.*]] = extractelement <4 x double> [[TMP28]], i64 0
-; I32-NEXT: store double [[TMP29]], ptr [[TMP46]], align 4
-; I32-NEXT: [[TMP30:%.*]] = extractelement <4 x double> [[TMP28]], i64 1
-; I32-NEXT: store double [[TMP30]], ptr [[TMP47]], align 4
-; I32-NEXT: [[TMP31:%.*]] = extractelement <4 x double> [[TMP28]], i64 2
-; I32-NEXT: store double [[TMP31]], ptr [[TMP48]], align 4
-; I32-NEXT: [[TMP32:%.*]] = extractelement <4 x double> [[TMP28]], i64 3
-; I32-NEXT: store double [[TMP32]], ptr [[TMP49]], align 4
-; I32-NEXT: [[TMP34:%.*]] = extractelement <4 x double> [[TMP33]], i64 0
-; I32-NEXT: store double [[TMP34]], ptr [[TMP50]], align 4
-; I32-NEXT: [[TMP35:%.*]] = extractelement <4 x double> [[TMP33]], i64 1
-; I32-NEXT: store double [[TMP35]], ptr [[TMP51]], align 4
-; I32-NEXT: [[TMP36:%.*]] = extractelement <4 x double> [[TMP33]], i64 2
-; I32-NEXT: store double [[TMP36]], ptr [[TMP52]], align 4
-; I32-NEXT: [[TMP37:%.*]] = extractelement <4 x double> [[TMP33]], i64 3
-; I32-NEXT: store double [[TMP37]], ptr [[TMP69]], align 4
-; I32-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; I32-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
-; I32-NEXT: [[TMP70:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; I32-NEXT: br i1 [[TMP70]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; I32: [[MIDDLE_BLOCK]]:
-; I32-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; I32: [[VEC_EPILOG_ITER_CHECK]]:
-; I32-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP2]], 4
-; I32-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
-; I32: [[VEC_EPILOG_PH]]:
-; I32-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; I32-NEXT: [[N_MOD_VF2:%.*]] = and i32 [[TMP0]], 3
-; I32-NEXT: [[TMP71:%.*]] = icmp eq i32 [[N_MOD_VF2]], 0
-; I32-NEXT: [[TMP72:%.*]] = select i1 [[TMP71]], i32 4, i32 [[N_MOD_VF2]]
-; I32-NEXT: [[N_VEC3:%.*]] = sub i32 [[TMP0]], [[TMP72]]
-; I32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; I32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; I32-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
-; I32-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; I32: [[VEC_EPILOG_VECTOR_BODY]]:
-; I32-NEXT: [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; I32-NEXT: [[VEC_IND5:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; I32-NEXT: [[TMP73:%.*]] = add i32 [[INDEX4]], 1
-; I32-NEXT: [[TMP74:%.*]] = add i32 [[INDEX4]], 2
-; I32-NEXT: [[TMP75:%.*]] = add i32 [[INDEX4]], 3
-; I32-NEXT: [[TMP76:%.*]] = uitofp <4 x i32> [[VEC_IND5]] to <4 x double>
-; I32-NEXT: [[TMP84:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX4]]
-; I32-NEXT: [[TMP85:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP73]]
-; I32-NEXT: [[TMP86:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP74]]
-; I32-NEXT: [[TMP87:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP75]]
; I32-NEXT: [[TMP81:%.*]] = load ptr, ptr [[TMP84]], align 4
; I32-NEXT: [[TMP82:%.*]] = load ptr, ptr [[TMP85]], align 4
; I32-NEXT: [[TMP83:%.*]] = load ptr, ptr [[TMP86]], align 4
; I32-NEXT: [[TMP88:%.*]] = load ptr, ptr [[TMP87]], align 4
-; I32-NEXT: [[TMP77:%.*]] = extractelement <4 x double> [[TMP76]], i64 0
+; I32-NEXT: [[TMP28:%.*]] = extractelement <8 x double> [[TMP11]], i64 0
+; I32-NEXT: store double [[TMP28]], ptr [[TMP50]], align 4
+; I32-NEXT: [[TMP29:%.*]] = extractelement <8 x double> [[TMP11]], i64 1
+; I32-NEXT: store double [[TMP29]], ptr [[TMP51]], align 4
+; I32-NEXT: [[TMP30:%.*]] = extractelement <8 x double> [[TMP11]], i64 2
+; I32-NEXT: store double [[TMP30]], ptr [[TMP52]], align 4
+; I32-NEXT: [[TMP31:%.*]] = extractelement <8 x double> [[TMP11]], i64 3
+; I32-NEXT: store double [[TMP31]], ptr [[TMP69]], align 4
+; I32-NEXT: [[TMP77:%.*]] = extractelement <8 x double> [[TMP11]], i64 4
; I32-NEXT: store double [[TMP77]], ptr [[TMP81]], align 4
-; I32-NEXT: [[TMP78:%.*]] = extractelement <4 x double> [[TMP76]], i64 1
+; I32-NEXT: [[TMP78:%.*]] = extractelement <8 x double> [[TMP11]], i64 5
; I32-NEXT: store double [[TMP78]], ptr [[TMP82]], align 4
-; I32-NEXT: [[TMP79:%.*]] = extractelement <4 x double> [[TMP76]], i64 2
+; I32-NEXT: [[TMP79:%.*]] = extractelement <8 x double> [[TMP11]], i64 6
; I32-NEXT: store double [[TMP79]], ptr [[TMP83]], align 4
-; I32-NEXT: [[TMP80:%.*]] = extractelement <4 x double> [[TMP76]], i64 3
+; I32-NEXT: [[TMP80:%.*]] = extractelement <8 x double> [[TMP11]], i64 7
; I32-NEXT: store double [[TMP80]], ptr [[TMP88]], align 4
-; I32-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX4]], 4
-; I32-NEXT: [[VEC_IND_NEXT7]] = add <4 x i32> [[VEC_IND5]], splat (i32 4)
-; I32-NEXT: [[TMP89:%.*]] = icmp eq i32 [[INDEX_NEXT6]], [[N_VEC3]]
-; I32-NEXT: br i1 [[TMP89]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; I32-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; I32-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; I32-NEXT: [[TMP36:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; I32-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; I32: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; I32-NEXT: br label %[[VEC_EPILOG_SCALAR_PH]]
-; I32: [[VEC_EPILOG_SCALAR_PH]]:
+; I32-NEXT: br label %[[VEC_EPILOG_PH]]
+; I32: [[VEC_EPILOG_PH]]:
;
entry:
br label %loop
@@ -419,7 +328,7 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I32-NEXT: store double [[TMP10]], ptr [[TMP18]], align 8
; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; I32-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; I32-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; I32-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
; I32-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC]]
; I32-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
@@ -817,7 +726,7 @@ define void @loaded_address_used_by_load_through_blend(i64 %start, ptr noalias %
; I32-NEXT: store float [[TMP82]], ptr [[TMP90]], align 4
; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; I32-NEXT: [[TMP91:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; I32-NEXT: br i1 [[TMP91]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; I32-NEXT: br i1 [[TMP91]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
; I32-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; I32-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -897,54 +806,102 @@ define void @address_use_in_different_block(ptr noalias %dst, ptr %src.0, ptr %s
; I32-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; I32-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
; I32-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; I32-NEXT: [[TMP31:%.*]] = add i64 [[INDEX]], 4
+; I32-NEXT: [[TMP32:%.*]] = add i64 [[INDEX]], 5
+; I32-NEXT: [[TMP33:%.*]] = add i64 [[INDEX]], 6
+; I32-NEXT: [[TMP34:%.*]] = add i64 [[INDEX]], 7
; I32-NEXT: [[TMP3:%.*]] = mul i64 [[INDEX]], [[OFFSET]]
; I32-NEXT: [[TMP4:%.*]] = mul i64 [[TMP0]], [[OFFSET]]
; I32-NEXT: [[TMP5:%.*]] = mul i64 [[TMP1]], [[OFFSET]]
; I32-NEXT: [[TMP6:%.*]] = mul i64 [[TMP2]], [[OFFSET]]
+; I32-NEXT: [[TMP47:%.*]] = mul i64 [[TMP31]], [[OFFSET]]
+; I32-NEXT: [[TMP48:%.*]] = mul i64 [[TMP32]], [[OFFSET]]
+; I32-NEXT: [[TMP49:%.*]] = mul i64 [[TMP33]], [[OFFSET]]
+; I32-NEXT: [[TMP50:%.*]] = mul i64 [[TMP34]], [[OFFSET]]
; I32-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP3]]
; I32-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP4]]
; I32-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP5]]
; I32-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP6]]
+; I32-NEXT: [[TMP55:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP47]]
+; I32-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP48]]
+; I32-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP49]]
+; I32-NEXT: [[TMP58:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP50]]
; I32-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
; I32-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP8]], align 4
; I32-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP9]], align 4
; I32-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP10]], align 4
+; I32-NEXT: [[TMP72:%.*]] = load i32, ptr [[TMP55]], align 4
+; I32-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP56]], align 4
+; I32-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP57]], align 4
+; I32-NEXT: [[TMP75:%.*]] = load i32, ptr [[TMP58]], align 4
; I32-NEXT: [[TMP15:%.*]] = sext i32 [[TMP11]] to i64
; I32-NEXT: [[TMP16:%.*]] = sext i32 [[TMP12]] to i64
; I32-NEXT: [[TMP17:%.*]] = sext i32 [[TMP13]] to i64
; I32-NEXT: [[TMP18:%.*]] = sext i32 [[TMP14]] to i64
+; I32-NEXT: [[TMP35:%.*]] = sext i32 [[TMP72]] to i64
+; I32-NEXT: [[TMP80:%.*]] = sext i32 [[TMP73]] to i64
+; I32-NEXT: [[TMP81:%.*]] = sext i32 [[TMP74]] to i64
+; I32-NEXT: [[TMP82:%.*]] = sext i32 [[TMP75]] to i64
; I32-NEXT: [[TMP19:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP15]]
; I32-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP16]]
; I32-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP17]]
; I32-NEXT: [[TMP22:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP18]]
+; I32-NEXT: [[TMP83:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP35]]
+; I32-NEXT: [[TMP44:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP80]]
+; I32-NEXT: [[TMP45:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP81]]
+; I32-NEXT: [[TMP46:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP82]]
; I32-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP19]], i64 -8
; I32-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP20]], i64 -8
; I32-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[TMP21]], i64 -8
; I32-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP22]], i64 -8
+; I32-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[TMP83]], i64 -8
+; I32-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[TMP44]], i64 -8
+; I32-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[TMP45]], i64 -8
+; I32-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[TMP46]], i64 -8
; I32-NEXT: [[TMP27:%.*]] = load double, ptr [[TMP23]], align 8
; I32-NEXT: [[TMP28:%.*]] = load double, ptr [[TMP24]], align 8
; I32-NEXT: [[TMP29:%.*]] = load double, ptr [[TMP25]], align 8
; I32-NEXT: [[TMP30:%.*]] = load double, ptr [[TMP26]], align 8
-; I32-NEXT: [[TMP31:%.*]] = insertelement <4 x double> poison, double [[TMP27]], i64 0
-; I32-NEXT: [[TMP32:%.*]] = insertelement <4 x double> [[TMP31]], double [[TMP28]], i64 1
-; I32-NEXT: [[TMP33:%.*]] = insertelement <4 x double> [[TMP32]], double [[TMP29]], i64 2
-; I32-NEXT: [[TMP34:%.*]] = insertelement <4 x double> [[TMP33]], double [[TMP30]], i64 3
-; I32-NEXT: [[TMP35:%.*]] = fsub <4 x double> zeroinitializer, [[TMP34]]
+; I32-NEXT: [[TMP59:%.*]] = load double, ptr [[TMP51]], align 8
+; I32-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP52]], align 8
+; I32-NEXT: [[TMP61:%.*]] = load double, ptr [[TMP53]], align 8
+; I32-NEXT: [[TMP62:%.*]] = load double, ptr [[TMP54]], align 8
+; I32-NEXT: [[TMP63:%.*]] = insertelement <8 x double> poison, double [[TMP27]], i64 0
+; I32-NEXT: [[TMP64:%.*]] = insertelement <8 x double> [[TMP63]], double [[TMP28]], i64 1
+; I32-NEXT: [[TMP65:%.*]] = insertelement <8 x double> [[TMP64]], double [[TMP29]], i64 2
+; I32-NEXT: [[TMP66:%.*]] = insertelement <8 x double> [[TMP65]], double [[TMP30]], i64 3
+; I32-NEXT: [[TMP67:%.*]] = insertelement <8 x double> [[TMP66]], double [[TMP59]], i64 4
+; I32-NEXT: [[TMP68:%.*]] = insertelement <8 x double> [[TMP67]], double [[TMP60]], i64 5
+; I32-NEXT: [[TMP69:%.*]] = insertelement <8 x double> [[TMP68]], double [[TMP61]], i64 6
+; I32-NEXT: [[TMP70:%.*]] = insertelement <8 x double> [[TMP69]], double [[TMP62]], i64 7
+; I32-NEXT: [[TMP71:%.*]] = fsub <8 x double> zeroinitializer, [[TMP70]]
; I32-NEXT: [[TMP40:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP3]]
; I32-NEXT: [[TMP41:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP4]]
; I32-NEXT: [[TMP42:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP5]]
; I32-NEXT: [[TMP43:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP6]]
-; I32-NEXT: [[TMP36:%.*]] = extractelement <4 x double> [[TMP35]], i64 0
+; I32-NEXT: [[TMP76:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP47]]
+; I32-NEXT: [[TMP77:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP48]]
+; I32-NEXT: [[TMP78:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP49]]
+; I32-NEXT: [[TMP79:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP50]]
+; I32-NEXT: [[TMP36:%.*]] = extractelement <8 x double> [[TMP71]], i64 0
; I32-NEXT: store double [[TMP36]], ptr [[TMP40]], align 8
-; I32-NEXT: [[TMP37:%.*]] = extractelement <4 x double> [[TMP35]], i64 1
+; I32-NEXT: [[TMP37:%.*]] = extractelement <8 x double> [[TMP71]], i64 1
; I32-NEXT: store double [[TMP37]], ptr [[TMP41]], align 8
-; I32-NEXT: [[TMP38:%.*]] = extractelement <4 x double> [[TMP35]], i64 2
+; I32-NEXT: [[TMP38:%.*]] = extractelement <8 x double> [[TMP71]], i64 2
; I32-NEXT: store double [[TMP38]], ptr [[TMP42]], align 8
-; I32-NEXT: [[TMP39:%.*]] = extractelement <4 x double> [[TMP35]], i64 3
+; I32-NEXT: [[TMP39:%.*]] = extractelement <8 x double> [[TMP71]], i64 3
; I32-NEXT: store double [[TMP39]], ptr [[TMP43]], align 8
-; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; I32-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; I32-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; I32-NEXT: [[TMP84:%.*]] = extractelement <8 x double> [[TMP71]], i64 4
+; I32-NEXT: store double [[TMP84]], ptr [[TMP76]], align 8
+; I32-NEXT: [[TMP85:%.*]] = extractelement <8 x double> [[TMP71]], i64 5
+; I32-NEXT: store double [[TMP85]], ptr [[TMP77]], align 8
+; I32-NEXT: [[TMP86:%.*]] = extractelement <8 x double> [[TMP71]], i64 6
+; I32-NEXT: store double [[TMP86]], ptr [[TMP78]], align 8
+; I32-NEXT: [[TMP87:%.*]] = extractelement <8 x double> [[TMP71]], i64 7
+; I32-NEXT: store double [[TMP87]], ptr [[TMP79]], align 8
+; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; I32-NEXT: [[TMP88:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; I32-NEXT: br i1 [[TMP88]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
; I32-NEXT: br label %[[SCALAR_PH:.*]]
; I32: [[SCALAR_PH]]:
@@ -1319,7 +1276,7 @@ define void @invariant_pred_store_sunk_out_of_loop(ptr noalias %dst, ptr noalias
; I32-NEXT: [[TMP5]] = add <2 x i64> [[TMP3]], splat (i64 1)
; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; I32-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; I32-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; I32-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
; I32-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP5]], [[TMP4]]
; I32-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
index 576a27bce5df4..4c5bf44ee7911 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -509,12 +509,36 @@ define void @test(ptr %A, ptr noalias %B) #0 {
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 10
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 12
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 14
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 18
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 20
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 22
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 24
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 26
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 28
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 30
+; CHECK-NEXT: [[TMP37:%.*]] = add i64 [[OFFSET_IDX]], 32
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], 34
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 36
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[OFFSET_IDX]], 38
+; CHECK-NEXT: [[TMP38:%.*]] = add i64 [[OFFSET_IDX]], 40
+; CHECK-NEXT: [[TMP39:%.*]] = add i64 [[OFFSET_IDX]], 42
+; CHECK-NEXT: [[TMP40:%.*]] = add i64 [[OFFSET_IDX]], 44
+; CHECK-NEXT: [[TMP41:%.*]] = add i64 [[OFFSET_IDX]], 46
+; CHECK-NEXT: [[TMP42:%.*]] = add i64 [[OFFSET_IDX]], 48
+; CHECK-NEXT: [[TMP67:%.*]] = add i64 [[OFFSET_IDX]], 50
+; CHECK-NEXT: [[TMP68:%.*]] = add i64 [[OFFSET_IDX]], 52
+; CHECK-NEXT: [[TMP69:%.*]] = add i64 [[OFFSET_IDX]], 54
+; CHECK-NEXT: [[TMP70:%.*]] = add i64 [[OFFSET_IDX]], 56
+; CHECK-NEXT: [[TMP71:%.*]] = add i64 [[OFFSET_IDX]], 58
+; CHECK-NEXT: [[TMP72:%.*]] = add i64 [[OFFSET_IDX]], 60
+; CHECK-NEXT: [[TMP73:%.*]] = add i64 [[OFFSET_IDX]], 62
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [1024 x i32], ptr [[A]], i64 0, i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP16]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT: [[TMP18:%.*]] = add <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP19:%.*]] = trunc <8 x i32> [[TMP18]] to <8 x i8>
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <64 x i32>, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
+; CHECK-NEXT: [[TMP74:%.*]] = add <32 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; CHECK-NEXT: [[TMP99:%.*]] = trunc <32 x i32> [[TMP74]] to <32 x i8>
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP1]]
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP2]]
@@ -523,23 +547,95 @@ define void @test(ptr %A, ptr noalias %B) #0 {
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP5]]
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP6]]
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP7]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <8 x i8> [[TMP19]], i64 0
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP9]]
+; CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP10]]
+; CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP11]]
+; CHECK-NEXT: [[TMP47:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP12]]
+; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP13]]
+; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP14]]
+; CHECK-NEXT: [[TMP50:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP15]]
+; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP37]]
+; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP17]]
+; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP18]]
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP19]]
+; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP38]]
+; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP39]]
+; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP40]]
+; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP41]]
+; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP42]]
+; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP67]]
+; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP68]]
+; CHECK-NEXT: [[TMP62:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP69]]
+; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP70]]
+; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP71]]
+; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP72]]
+; CHECK-NEXT: [[TMP66:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP73]]
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <32 x i8> [[TMP99]], i64 0
; CHECK-NEXT: store i8 [[TMP28]], ptr [[TMP20]], align 1
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <8 x i8> [[TMP19]], i64 1
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <32 x i8> [[TMP99]], i64 1
; CHECK-NEXT: store i8 [[TMP29]], ptr [[TMP21]], align 1
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <8 x i8> [[TMP19]], i64 2
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <32 x i8> [[TMP99]], i64 2
; CHECK-NEXT: store i8 [[TMP30]], ptr [[TMP22]], align 1
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <8 x i8> [[TMP19]], i64 3
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <32 x i8> [[TMP99]], i64 3
; CHECK-NEXT: store i8 [[TMP31]], ptr [[TMP23]], align 1
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <8 x i8> [[TMP19]], i64 4
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <32 x i8> [[TMP99]], i64 4
; CHECK-NEXT: store i8 [[TMP32]], ptr [[TMP24]], align 1
-; CHECK-NEXT: [[TMP33:%.*]] = extractelement <8 x i8> [[TMP19]], i64 5
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <32 x i8> [[TMP99]], i64 5
; CHECK-NEXT: store i8 [[TMP33]], ptr [[TMP25]], align 1
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <8 x i8> [[TMP19]], i64 6
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <32 x i8> [[TMP99]], i64 6
; CHECK-NEXT: store i8 [[TMP34]], ptr [[TMP26]], align 1
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <8 x i8> [[TMP19]], i64 7
+; CHECK-NEXT: [[TMP35:%.*]] = extractelement <32 x i8> [[TMP99]], i64 7
; CHECK-NEXT: store i8 [[TMP35]], ptr [[TMP27]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP75:%.*]] = extractelement <32 x i8> [[TMP99]], i64 8
+; CHECK-NEXT: store i8 [[TMP75]], ptr [[TMP43]], align 1
+; CHECK-NEXT: [[TMP76:%.*]] = extractelement <32 x i8> [[TMP99]], i64 9
+; CHECK-NEXT: store i8 [[TMP76]], ptr [[TMP44]], align 1
+; CHECK-NEXT: [[TMP77:%.*]] = extractelement <32 x i8> [[TMP99]], i64 10
+; CHECK-NEXT: store i8 [[TMP77]], ptr [[TMP45]], align 1
+; CHECK-NEXT: [[TMP78:%.*]] = extractelement <32 x i8> [[TMP99]], i64 11
+; CHECK-NEXT: store i8 [[TMP78]], ptr [[TMP46]], align 1
+; CHECK-NEXT: [[TMP79:%.*]] = extractelement <32 x i8> [[TMP99]], i64 12
+; CHECK-NEXT: store i8 [[TMP79]], ptr [[TMP47]], align 1
+; CHECK-NEXT: [[TMP80:%.*]] = extractelement <32 x i8> [[TMP99]], i64 13
+; CHECK-NEXT: store i8 [[TMP80]], ptr [[TMP48]], align 1
+; CHECK-NEXT: [[TMP81:%.*]] = extractelement <32 x i8> [[TMP99]], i64 14
+; CHECK-NEXT: store i8 [[TMP81]], ptr [[TMP49]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <32 x i8> [[TMP99]], i64 15
+; CHECK-NEXT: store i8 [[TMP82]], ptr [[TMP50]], align 1
+; CHECK-NEXT: [[TMP83:%.*]] = extractelement <32 x i8> [[TMP99]], i64 16
+; CHECK-NEXT: store i8 [[TMP83]], ptr [[TMP51]], align 1
+; CHECK-NEXT: [[TMP84:%.*]] = extractelement <32 x i8> [[TMP99]], i64 17
+; CHECK-NEXT: store i8 [[TMP84]], ptr [[TMP52]], align 1
+; CHECK-NEXT: [[TMP85:%.*]] = extractelement <32 x i8> [[TMP99]], i64 18
+; CHECK-NEXT: store i8 [[TMP85]], ptr [[TMP53]], align 1
+; CHECK-NEXT: [[TMP86:%.*]] = extractelement <32 x i8> [[TMP99]], i64 19
+; CHECK-NEXT: store i8 [[TMP86]], ptr [[TMP54]], align 1
+; CHECK-NEXT: [[TMP87:%.*]] = extractelement <32 x i8> [[TMP99]], i64 20
+; CHECK-NEXT: store i8 [[TMP87]], ptr [[TMP55]], align 1
+; CHECK-NEXT: [[TMP88:%.*]] = extractelement <32 x i8> [[TMP99]], i64 21
+; CHECK-NEXT: store i8 [[TMP88]], ptr [[TMP56]], align 1
+; CHECK-NEXT: [[TMP89:%.*]] = extractelement <32 x i8> [[TMP99]], i64 22
+; CHECK-NEXT: store i8 [[TMP89]], ptr [[TMP57]], align 1
+; CHECK-NEXT: [[TMP90:%.*]] = extractelement <32 x i8> [[TMP99]], i64 23
+; CHECK-NEXT: store i8 [[TMP90]], ptr [[TMP58]], align 1
+; CHECK-NEXT: [[TMP91:%.*]] = extractelement <32 x i8> [[TMP99]], i64 24
+; CHECK-NEXT: store i8 [[TMP91]], ptr [[TMP59]], align 1
+; CHECK-NEXT: [[TMP92:%.*]] = extractelement <32 x i8> [[TMP99]], i64 25
+; CHECK-NEXT: store i8 [[TMP92]], ptr [[TMP60]], align 1
+; CHECK-NEXT: [[TMP93:%.*]] = extractelement <32 x i8> [[TMP99]], i64 26
+; CHECK-NEXT: store i8 [[TMP93]], ptr [[TMP61]], align 1
+; CHECK-NEXT: [[TMP94:%.*]] = extractelement <32 x i8> [[TMP99]], i64 27
+; CHECK-NEXT: store i8 [[TMP94]], ptr [[TMP62]], align 1
+; CHECK-NEXT: [[TMP95:%.*]] = extractelement <32 x i8> [[TMP99]], i64 28
+; CHECK-NEXT: store i8 [[TMP95]], ptr [[TMP63]], align 1
+; CHECK-NEXT: [[TMP96:%.*]] = extractelement <32 x i8> [[TMP99]], i64 29
+; CHECK-NEXT: store i8 [[TMP96]], ptr [[TMP64]], align 1
+; CHECK-NEXT: [[TMP97:%.*]] = extractelement <32 x i8> [[TMP99]], i64 30
+; CHECK-NEXT: store i8 [[TMP97]], ptr [[TMP65]], align 1
+; CHECK-NEXT: [[TMP98:%.*]] = extractelement <32 x i8> [[TMP99]], i64 31
+; CHECK-NEXT: store i8 [[TMP98]], ptr [[TMP66]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
; CHECK-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
index 91907c3e4d69e..fe886aac6f9b5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
@@ -96,7 +96,7 @@ define void @test_nonconsecutive_store() {
;; pointer types into account.
; CHECK: test_consecutive_ptr_load
; CHECK: LV: The Smallest and Widest types: 8 / 64 bits.
-; CHECK: LV: Selecting VF: 4
+; CHECK: LV: Selecting VF: 16
define i8 @test_consecutive_ptr_load() readonly {
br label %1
@@ -121,7 +121,7 @@ define i8 @test_consecutive_ptr_load() readonly {
;; However, we should not take unconsecutive loads of pointers into account.
; CHECK: test_nonconsecutive_ptr_load
; CHECK: LV: The Smallest and Widest types: 16 / 64 bits.
-; CHECK: LV: Selecting VF: 1
+; CHECK: LV: Selecting VF: 16
define void @test_nonconsecutive_ptr_load() {
br label %1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll
index 9ded7fa7d6a3c..1c162d062cc15 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll
@@ -2,7 +2,7 @@
; RUN: opt < %s -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -mtriple=x86_64-unknown-linux -S -pass-remarks='loop-vectorize' 2>&1 | FileCheck -check-prefix=UNROLLED %s
; RUN: opt < %s -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=1 -mtriple=x86_64-unknown-linux -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck -check-prefix=NONE %s
-; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 4, interleaved count: 2)
+; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 16, interleaved count: 1)
; UNROLLED: remark: vectorization-remarks.c:17:8: interleaved loop (interleaved count: 4)
; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
index 41ad9ec20cc3d..c4cbed28381bc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
@@ -2,7 +2,7 @@
; RUN: opt < %s -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -mtriple=x86_64-unknown-linux -S -pass-remarks='loop-vectorize' 2>&1 | FileCheck -check-prefix=UNROLLED %s
; RUN: opt < %s -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=1 -mtriple=x86_64-unknown-linux -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck -check-prefix=NONE %s
-; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 4, interleaved count: 2)
+; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 16, interleaved count: 1)
; UNROLLED: remark: vectorization-remarks.c:17:8: interleaved loop (interleaved count: 4)
; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll b/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
index 975de14a95fc4..d168f04485caf 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
@@ -23,37 +23,56 @@ define void @loop_or(ptr noalias %pIn, ptr noalias %pOut, i32 %s) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[S:%.*]], 0
; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; CHECK: for.body.preheader:
+; CHECK: iter.check:
; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[S]] to i64
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[S]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[S]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER5:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.main.loop.iter.check:
+; CHECK-NEXT: [[MIN_ITERS_CHECK4:%.*]] = icmp ult i32 [[S]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK4]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483640
+; CHECK-NEXT: [[TMP8:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 12
+; CHECK-NEXT: [[N_VEC1:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483632
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[PIN:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP12:%.*]] = mul nuw nsw <16 x i32> [[TMP2]], splat (i32 65793)
+; CHECK-NEXT: [[TMP4:%.*]] = or disjoint <16 x i32> [[TMP12]], splat (i32 -16777216)
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[POUT:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: store <16 x i32> [[TMP4]], ptr [[TMP13]], align 4
+; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC1]]
+; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N_VEC1]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[CMP_N1]], label [[FOR_END]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK: vec.epilog.iter.check:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[TMP8]], 0
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[FOR_BODY_PREHEADER5]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: vec.epilog.ph:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483644
+; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK: vec.epilog.vector.body:
+; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[PIN]], i64 [[INDEX6]]
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[WIDE_LOAD4]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[TMP2]], splat (i32 65793)
; CHECK-NEXT: [[TMP5:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 65793)
-; CHECK-NEXT: [[TMP6:%.*]] = or disjoint <4 x i32> [[TMP4]], splat (i32 -16777216)
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint <4 x i32> [[TMP5]], splat (i32 -16777216)
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[POUT:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
-; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[POUT]], i64 [[INDEX6]]
; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX6]], 4
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER5]]
-; CHECK: for.body.preheader5:
-; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER5]] ]
@@ -66,7 +85,7 @@ define void @loop_or(ptr noalias %pIn, ptr noalias %pOut, i32 %s) {
; CHECK-NEXT: store i32 [[OR3]], ptr [[ARRAYIDX5]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: for.end:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
index a66528f1f12b4..9ca53b494de2d 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
@@ -15,27 +15,27 @@ define void @test(i32 noundef %nface, i32 noundef %ncell, ptr noalias noundef %f
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[NFACE]] to i64
; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[FACE_CELL]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[NFACE]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[NFACE]], 8
; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_BODY_PREHEADER14:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[TMP0]], 2147483644
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[TMP0]], 2147483640
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[FACE_CELL]], i64 [[INDVARS_IV_EPIL]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !llvm.access.group [[ACC_GRP4:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !llvm.access.group [[ACC_GRP4:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[INVARIANT_GEP]], i64 [[INDVARS_IV_EPIL]]
-; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !tbaa [[INT_TBAA0]], !llvm.access.group [[ACC_GRP4]]
-; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [8 x i8], ptr [[Y]], <4 x i64> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i32> [[WIDE_LOAD12]] to <4 x i64>
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <4 x i64> [[TMP5]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = tail call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP4]], <4 x i1> splat (i1 true), <4 x double> poison), !tbaa [[DOUBLE_TBAA5:![0-9]+]], !llvm.access.group [[ACC_GRP4]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER13:%.*]] = tail call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP6]], <4 x i1> splat (i1 true), <4 x double> poison), !tbaa [[DOUBLE_TBAA5]], !llvm.access.group [[ACC_GRP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast olt <4 x double> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER13]]
-; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x double> [[WIDE_MASKED_GATHER13]], <4 x double> [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: tail call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP8]], <4 x ptr> align 8 [[TMP4]], <4 x i1> splat (i1 true)), !tbaa [[DOUBLE_TBAA5]], !llvm.access.group [[ACC_GRP4]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV_EPIL]], 4
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4, !tbaa [[INT_TBAA0]], !llvm.access.group [[ACC_GRP4]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i32> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [8 x i8], ptr [[Y]], <8 x i64> [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i32> [[WIDE_LOAD12]] to <8 x i64>
+; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <8 x i64> [[TMP4]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = tail call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[WIDE_GEP]], <8 x i1> splat (i1 true), <8 x double> poison), !tbaa [[DOUBLE_TBAA5:![0-9]+]], !llvm.access.group [[ACC_GRP4]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER14:%.*]] = tail call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[WIDE_GEP13]], <8 x i1> splat (i1 true), <8 x double> poison), !tbaa [[DOUBLE_TBAA5]], !llvm.access.group [[ACC_GRP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp fast olt <8 x double> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER14]]
+; CHECK-NEXT: [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x double> [[WIDE_MASKED_GATHER14]], <8 x double> [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: tail call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> [[TMP6]], <8 x ptr> align 8 [[WIDE_GEP]], <8 x i1> splat (i1 true)), !tbaa [[DOUBLE_TBAA5]], !llvm.access.group [[ACC_GRP4]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV_EPIL]], 8
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll
index 0f15bbe5a69f8..61a8969fad60c 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll
@@ -11,295 +11,394 @@ define i16 @test(ptr %ptr) {
; SSE-NEXT: entry:
; SSE-NEXT: [[FIRST:%.*]] = load i8, ptr [[PTR:%.*]], align 1
; SSE-NEXT: tail call void @use(i8 [[FIRST]]) #[[ATTR3:[0-9]+]]
-; SSE-NEXT: br label [[VECTOR_BODY:%.*]]
-; SSE: vector.body:
-; SSE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ]
-; SSE-NEXT: [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[ENTRY]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
-; SSE-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i16> [ zeroinitializer, [[ENTRY]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; SSE-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[INDEX]]
-; SSE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
-; SSE-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; SSE-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; SSE-NEXT: [[TMP2:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i16>
-; SSE-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i16>
-; SSE-NEXT: [[TMP4:%.*]] = add <8 x i16> [[VEC_PHI]], [[TMP2]]
-; SSE-NEXT: [[TMP5:%.*]] = add <8 x i16> [[VEC_PHI1]], [[TMP3]]
-; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[INDEX]]
-; SSE-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 16
-; SSE-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 24
-; SSE-NEXT: [[WIDE_LOAD_1:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
-; SSE-NEXT: [[WIDE_LOAD2_1:%.*]] = load <8 x i8>, ptr [[TMP8]], align 1
-; SSE-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_LOAD_1]] to <8 x i16>
-; SSE-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_LOAD2_1]] to <8 x i16>
-; SSE-NEXT: [[TMP11]] = add <8 x i16> [[TMP4]], [[TMP9]]
-; SSE-NEXT: [[TMP12]] = add <8 x i16> [[TMP5]], [[TMP10]]
-; SSE-NEXT: [[INDEX_NEXT_1]] = add nuw nsw i64 [[INDEX]], 32
-; SSE-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], 1024
-; SSE-NEXT: br i1 [[TMP13]], label [[EXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; SSE: exit:
-; SSE-NEXT: [[BIN_RDX:%.*]] = add <8 x i16> [[TMP12]], [[TMP11]]
-; SSE-NEXT: [[TMP14:%.*]] = tail call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[BIN_RDX]])
+; SSE-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 16
+; SSE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[PTR]], align 1
+; SSE-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; SSE-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i16>
+; SSE-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i16>
+; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 32
+; SSE-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 48
+; SSE-NEXT: [[WIDE_LOAD_1:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_1:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; SSE-NEXT: [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD_1]] to <16 x i16>
+; SSE-NEXT: [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD2_1]] to <16 x i16>
+; SSE-NEXT: [[TMP7:%.*]] = add nuw nsw <16 x i16> [[TMP1]], [[TMP5]]
+; SSE-NEXT: [[TMP8:%.*]] = add nuw nsw <16 x i16> [[TMP2]], [[TMP6]]
+; SSE-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 64
+; SSE-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 80
+; SSE-NEXT: [[WIDE_LOAD_2:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_2:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1
+; SSE-NEXT: [[TMP11:%.*]] = zext <16 x i8> [[WIDE_LOAD_2]] to <16 x i16>
+; SSE-NEXT: [[TMP12:%.*]] = zext <16 x i8> [[WIDE_LOAD2_2]] to <16 x i16>
+; SSE-NEXT: [[TMP13:%.*]] = add nuw nsw <16 x i16> [[TMP7]], [[TMP11]]
+; SSE-NEXT: [[TMP189:%.*]] = add nuw nsw <16 x i16> [[TMP8]], [[TMP12]]
+; SSE-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 96
+; SSE-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 112
+; SSE-NEXT: [[WIDE_LOAD_3:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_3:%.*]] = load <16 x i8>, ptr [[TMP16]], align 1
+; SSE-NEXT: [[TMP17:%.*]] = zext <16 x i8> [[WIDE_LOAD_3]] to <16 x i16>
+; SSE-NEXT: [[TMP18:%.*]] = zext <16 x i8> [[WIDE_LOAD2_3]] to <16 x i16>
+; SSE-NEXT: [[TMP19:%.*]] = add nuw nsw <16 x i16> [[TMP13]], [[TMP17]]
+; SSE-NEXT: [[TMP20:%.*]] = add nuw nsw <16 x i16> [[TMP189]], [[TMP18]]
+; SSE-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 128
+; SSE-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 144
+; SSE-NEXT: [[WIDE_LOAD_4:%.*]] = load <16 x i8>, ptr [[TMP21]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_4:%.*]] = load <16 x i8>, ptr [[TMP22]], align 1
+; SSE-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD_4]] to <16 x i16>
+; SSE-NEXT: [[TMP24:%.*]] = zext <16 x i8> [[WIDE_LOAD2_4]] to <16 x i16>
+; SSE-NEXT: [[TMP25:%.*]] = add nuw nsw <16 x i16> [[TMP19]], [[TMP23]]
+; SSE-NEXT: [[TMP26:%.*]] = add nuw nsw <16 x i16> [[TMP20]], [[TMP24]]
+; SSE-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 160
+; SSE-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 176
+; SSE-NEXT: [[WIDE_LOAD_5:%.*]] = load <16 x i8>, ptr [[TMP27]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_5:%.*]] = load <16 x i8>, ptr [[TMP28]], align 1
+; SSE-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[WIDE_LOAD_5]] to <16 x i16>
+; SSE-NEXT: [[TMP30:%.*]] = zext <16 x i8> [[WIDE_LOAD2_5]] to <16 x i16>
+; SSE-NEXT: [[TMP31:%.*]] = add nuw nsw <16 x i16> [[TMP25]], [[TMP29]]
+; SSE-NEXT: [[TMP32:%.*]] = add nuw nsw <16 x i16> [[TMP26]], [[TMP30]]
+; SSE-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 192
+; SSE-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 208
+; SSE-NEXT: [[WIDE_LOAD_6:%.*]] = load <16 x i8>, ptr [[TMP33]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_6:%.*]] = load <16 x i8>, ptr [[TMP34]], align 1
+; SSE-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD_6]] to <16 x i16>
+; SSE-NEXT: [[TMP36:%.*]] = zext <16 x i8> [[WIDE_LOAD2_6]] to <16 x i16>
+; SSE-NEXT: [[TMP37:%.*]] = add nuw nsw <16 x i16> [[TMP31]], [[TMP35]]
+; SSE-NEXT: [[TMP38:%.*]] = add nuw nsw <16 x i16> [[TMP32]], [[TMP36]]
+; SSE-NEXT: [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 224
+; SSE-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 240
+; SSE-NEXT: [[WIDE_LOAD_7:%.*]] = load <16 x i8>, ptr [[TMP39]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_7:%.*]] = load <16 x i8>, ptr [[TMP40]], align 1
+; SSE-NEXT: [[TMP41:%.*]] = zext <16 x i8> [[WIDE_LOAD_7]] to <16 x i16>
+; SSE-NEXT: [[TMP42:%.*]] = zext <16 x i8> [[WIDE_LOAD2_7]] to <16 x i16>
+; SSE-NEXT: [[TMP43:%.*]] = add <16 x i16> [[TMP37]], [[TMP41]]
+; SSE-NEXT: [[TMP44:%.*]] = add <16 x i16> [[TMP38]], [[TMP42]]
+; SSE-NEXT: [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 256
+; SSE-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 272
+; SSE-NEXT: [[WIDE_LOAD_8:%.*]] = load <16 x i8>, ptr [[TMP45]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_8:%.*]] = load <16 x i8>, ptr [[TMP46]], align 1
+; SSE-NEXT: [[TMP47:%.*]] = zext <16 x i8> [[WIDE_LOAD_8]] to <16 x i16>
+; SSE-NEXT: [[TMP48:%.*]] = zext <16 x i8> [[WIDE_LOAD2_8]] to <16 x i16>
+; SSE-NEXT: [[TMP49:%.*]] = add <16 x i16> [[TMP43]], [[TMP47]]
+; SSE-NEXT: [[TMP50:%.*]] = add <16 x i16> [[TMP44]], [[TMP48]]
+; SSE-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 288
+; SSE-NEXT: [[TMP52:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 304
+; SSE-NEXT: [[WIDE_LOAD_9:%.*]] = load <16 x i8>, ptr [[TMP51]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_9:%.*]] = load <16 x i8>, ptr [[TMP52]], align 1
+; SSE-NEXT: [[TMP53:%.*]] = zext <16 x i8> [[WIDE_LOAD_9]] to <16 x i16>
+; SSE-NEXT: [[TMP54:%.*]] = zext <16 x i8> [[WIDE_LOAD2_9]] to <16 x i16>
+; SSE-NEXT: [[TMP55:%.*]] = add <16 x i16> [[TMP49]], [[TMP53]]
+; SSE-NEXT: [[TMP56:%.*]] = add <16 x i16> [[TMP50]], [[TMP54]]
+; SSE-NEXT: [[TMP57:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 320
+; SSE-NEXT: [[TMP58:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
+; SSE-NEXT: [[WIDE_LOAD_10:%.*]] = load <16 x i8>, ptr [[TMP57]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_10:%.*]] = load <16 x i8>, ptr [[TMP58]], align 1
+; SSE-NEXT: [[TMP59:%.*]] = zext <16 x i8> [[WIDE_LOAD_10]] to <16 x i16>
+; SSE-NEXT: [[TMP60:%.*]] = zext <16 x i8> [[WIDE_LOAD2_10]] to <16 x i16>
+; SSE-NEXT: [[TMP61:%.*]] = add <16 x i16> [[TMP55]], [[TMP59]]
+; SSE-NEXT: [[TMP62:%.*]] = add <16 x i16> [[TMP56]], [[TMP60]]
+; SSE-NEXT: [[TMP63:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 352
+; SSE-NEXT: [[TMP64:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 368
+; SSE-NEXT: [[WIDE_LOAD_11:%.*]] = load <16 x i8>, ptr [[TMP63]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_11:%.*]] = load <16 x i8>, ptr [[TMP64]], align 1
+; SSE-NEXT: [[TMP65:%.*]] = zext <16 x i8> [[WIDE_LOAD_11]] to <16 x i16>
+; SSE-NEXT: [[TMP66:%.*]] = zext <16 x i8> [[WIDE_LOAD2_11]] to <16 x i16>
+; SSE-NEXT: [[TMP67:%.*]] = add <16 x i16> [[TMP61]], [[TMP65]]
+; SSE-NEXT: [[TMP68:%.*]] = add <16 x i16> [[TMP62]], [[TMP66]]
+; SSE-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 384
+; SSE-NEXT: [[TMP70:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 400
+; SSE-NEXT: [[WIDE_LOAD_12:%.*]] = load <16 x i8>, ptr [[TMP69]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_12:%.*]] = load <16 x i8>, ptr [[TMP70]], align 1
+; SSE-NEXT: [[TMP71:%.*]] = zext <16 x i8> [[WIDE_LOAD_12]] to <16 x i16>
+; SSE-NEXT: [[TMP72:%.*]] = zext <16 x i8> [[WIDE_LOAD2_12]] to <16 x i16>
+; SSE-NEXT: [[TMP73:%.*]] = add <16 x i16> [[TMP67]], [[TMP71]]
+; SSE-NEXT: [[TMP74:%.*]] = add <16 x i16> [[TMP68]], [[TMP72]]
+; SSE-NEXT: [[TMP75:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 416
+; SSE-NEXT: [[TMP76:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 432
+; SSE-NEXT: [[WIDE_LOAD_13:%.*]] = load <16 x i8>, ptr [[TMP75]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_13:%.*]] = load <16 x i8>, ptr [[TMP76]], align 1
+; SSE-NEXT: [[TMP77:%.*]] = zext <16 x i8> [[WIDE_LOAD_13]] to <16 x i16>
+; SSE-NEXT: [[TMP78:%.*]] = zext <16 x i8> [[WIDE_LOAD2_13]] to <16 x i16>
+; SSE-NEXT: [[TMP79:%.*]] = add <16 x i16> [[TMP73]], [[TMP77]]
+; SSE-NEXT: [[TMP80:%.*]] = add <16 x i16> [[TMP74]], [[TMP78]]
+; SSE-NEXT: [[TMP81:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 448
+; SSE-NEXT: [[TMP82:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 464
+; SSE-NEXT: [[WIDE_LOAD_14:%.*]] = load <16 x i8>, ptr [[TMP81]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_14:%.*]] = load <16 x i8>, ptr [[TMP82]], align 1
+; SSE-NEXT: [[TMP83:%.*]] = zext <16 x i8> [[WIDE_LOAD_14]] to <16 x i16>
+; SSE-NEXT: [[TMP84:%.*]] = zext <16 x i8> [[WIDE_LOAD2_14]] to <16 x i16>
+; SSE-NEXT: [[TMP85:%.*]] = add <16 x i16> [[TMP79]], [[TMP83]]
+; SSE-NEXT: [[TMP86:%.*]] = add <16 x i16> [[TMP80]], [[TMP84]]
+; SSE-NEXT: [[TMP87:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 480
+; SSE-NEXT: [[TMP88:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 496
+; SSE-NEXT: [[WIDE_LOAD_15:%.*]] = load <16 x i8>, ptr [[TMP87]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_15:%.*]] = load <16 x i8>, ptr [[TMP88]], align 1
+; SSE-NEXT: [[TMP89:%.*]] = zext <16 x i8> [[WIDE_LOAD_15]] to <16 x i16>
+; SSE-NEXT: [[TMP90:%.*]] = zext <16 x i8> [[WIDE_LOAD2_15]] to <16 x i16>
+; SSE-NEXT: [[TMP91:%.*]] = add <16 x i16> [[TMP85]], [[TMP89]]
+; SSE-NEXT: [[TMP92:%.*]] = add <16 x i16> [[TMP86]], [[TMP90]]
+; SSE-NEXT: [[TMP93:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 512
+; SSE-NEXT: [[TMP94:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 528
+; SSE-NEXT: [[WIDE_LOAD_16:%.*]] = load <16 x i8>, ptr [[TMP93]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_16:%.*]] = load <16 x i8>, ptr [[TMP94]], align 1
+; SSE-NEXT: [[TMP95:%.*]] = zext <16 x i8> [[WIDE_LOAD_16]] to <16 x i16>
+; SSE-NEXT: [[TMP96:%.*]] = zext <16 x i8> [[WIDE_LOAD2_16]] to <16 x i16>
+; SSE-NEXT: [[TMP97:%.*]] = add <16 x i16> [[TMP91]], [[TMP95]]
+; SSE-NEXT: [[TMP98:%.*]] = add <16 x i16> [[TMP92]], [[TMP96]]
+; SSE-NEXT: [[TMP99:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 544
+; SSE-NEXT: [[TMP100:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 560
+; SSE-NEXT: [[WIDE_LOAD_17:%.*]] = load <16 x i8>, ptr [[TMP99]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_17:%.*]] = load <16 x i8>, ptr [[TMP100]], align 1
+; SSE-NEXT: [[TMP101:%.*]] = zext <16 x i8> [[WIDE_LOAD_17]] to <16 x i16>
+; SSE-NEXT: [[TMP102:%.*]] = zext <16 x i8> [[WIDE_LOAD2_17]] to <16 x i16>
+; SSE-NEXT: [[TMP103:%.*]] = add <16 x i16> [[TMP97]], [[TMP101]]
+; SSE-NEXT: [[TMP104:%.*]] = add <16 x i16> [[TMP98]], [[TMP102]]
+; SSE-NEXT: [[TMP105:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 576
+; SSE-NEXT: [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 592
+; SSE-NEXT: [[WIDE_LOAD_18:%.*]] = load <16 x i8>, ptr [[TMP105]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_18:%.*]] = load <16 x i8>, ptr [[TMP106]], align 1
+; SSE-NEXT: [[TMP107:%.*]] = zext <16 x i8> [[WIDE_LOAD_18]] to <16 x i16>
+; SSE-NEXT: [[TMP108:%.*]] = zext <16 x i8> [[WIDE_LOAD2_18]] to <16 x i16>
+; SSE-NEXT: [[TMP109:%.*]] = add <16 x i16> [[TMP103]], [[TMP107]]
+; SSE-NEXT: [[TMP110:%.*]] = add <16 x i16> [[TMP104]], [[TMP108]]
+; SSE-NEXT: [[TMP111:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 608
+; SSE-NEXT: [[TMP112:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 624
+; SSE-NEXT: [[WIDE_LOAD_19:%.*]] = load <16 x i8>, ptr [[TMP111]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_19:%.*]] = load <16 x i8>, ptr [[TMP112]], align 1
+; SSE-NEXT: [[TMP113:%.*]] = zext <16 x i8> [[WIDE_LOAD_19]] to <16 x i16>
+; SSE-NEXT: [[TMP114:%.*]] = zext <16 x i8> [[WIDE_LOAD2_19]] to <16 x i16>
+; SSE-NEXT: [[TMP115:%.*]] = add <16 x i16> [[TMP109]], [[TMP113]]
+; SSE-NEXT: [[TMP116:%.*]] = add <16 x i16> [[TMP110]], [[TMP114]]
+; SSE-NEXT: [[TMP117:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 640
+; SSE-NEXT: [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 656
+; SSE-NEXT: [[WIDE_LOAD_20:%.*]] = load <16 x i8>, ptr [[TMP117]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_20:%.*]] = load <16 x i8>, ptr [[TMP118]], align 1
+; SSE-NEXT: [[TMP119:%.*]] = zext <16 x i8> [[WIDE_LOAD_20]] to <16 x i16>
+; SSE-NEXT: [[TMP120:%.*]] = zext <16 x i8> [[WIDE_LOAD2_20]] to <16 x i16>
+; SSE-NEXT: [[TMP121:%.*]] = add <16 x i16> [[TMP115]], [[TMP119]]
+; SSE-NEXT: [[TMP122:%.*]] = add <16 x i16> [[TMP116]], [[TMP120]]
+; SSE-NEXT: [[TMP123:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 672
+; SSE-NEXT: [[TMP124:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 688
+; SSE-NEXT: [[WIDE_LOAD_21:%.*]] = load <16 x i8>, ptr [[TMP123]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_21:%.*]] = load <16 x i8>, ptr [[TMP124]], align 1
+; SSE-NEXT: [[TMP125:%.*]] = zext <16 x i8> [[WIDE_LOAD_21]] to <16 x i16>
+; SSE-NEXT: [[TMP126:%.*]] = zext <16 x i8> [[WIDE_LOAD2_21]] to <16 x i16>
+; SSE-NEXT: [[TMP127:%.*]] = add <16 x i16> [[TMP121]], [[TMP125]]
+; SSE-NEXT: [[TMP128:%.*]] = add <16 x i16> [[TMP122]], [[TMP126]]
+; SSE-NEXT: [[TMP129:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 704
+; SSE-NEXT: [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 720
+; SSE-NEXT: [[WIDE_LOAD_22:%.*]] = load <16 x i8>, ptr [[TMP129]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_22:%.*]] = load <16 x i8>, ptr [[TMP130]], align 1
+; SSE-NEXT: [[TMP131:%.*]] = zext <16 x i8> [[WIDE_LOAD_22]] to <16 x i16>
+; SSE-NEXT: [[TMP132:%.*]] = zext <16 x i8> [[WIDE_LOAD2_22]] to <16 x i16>
+; SSE-NEXT: [[TMP133:%.*]] = add <16 x i16> [[TMP127]], [[TMP131]]
+; SSE-NEXT: [[TMP134:%.*]] = add <16 x i16> [[TMP128]], [[TMP132]]
+; SSE-NEXT: [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 736
+; SSE-NEXT: [[TMP136:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 752
+; SSE-NEXT: [[WIDE_LOAD_23:%.*]] = load <16 x i8>, ptr [[TMP135]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_23:%.*]] = load <16 x i8>, ptr [[TMP136]], align 1
+; SSE-NEXT: [[TMP137:%.*]] = zext <16 x i8> [[WIDE_LOAD_23]] to <16 x i16>
+; SSE-NEXT: [[TMP138:%.*]] = zext <16 x i8> [[WIDE_LOAD2_23]] to <16 x i16>
+; SSE-NEXT: [[TMP139:%.*]] = add <16 x i16> [[TMP133]], [[TMP137]]
+; SSE-NEXT: [[TMP140:%.*]] = add <16 x i16> [[TMP134]], [[TMP138]]
+; SSE-NEXT: [[TMP141:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 768
+; SSE-NEXT: [[TMP142:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 784
+; SSE-NEXT: [[WIDE_LOAD_24:%.*]] = load <16 x i8>, ptr [[TMP141]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_24:%.*]] = load <16 x i8>, ptr [[TMP142]], align 1
+; SSE-NEXT: [[TMP143:%.*]] = zext <16 x i8> [[WIDE_LOAD_24]] to <16 x i16>
+; SSE-NEXT: [[TMP144:%.*]] = zext <16 x i8> [[WIDE_LOAD2_24]] to <16 x i16>
+; SSE-NEXT: [[TMP145:%.*]] = add <16 x i16> [[TMP139]], [[TMP143]]
+; SSE-NEXT: [[TMP146:%.*]] = add <16 x i16> [[TMP140]], [[TMP144]]
+; SSE-NEXT: [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 800
+; SSE-NEXT: [[TMP148:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 816
+; SSE-NEXT: [[WIDE_LOAD_25:%.*]] = load <16 x i8>, ptr [[TMP147]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_25:%.*]] = load <16 x i8>, ptr [[TMP148]], align 1
+; SSE-NEXT: [[TMP149:%.*]] = zext <16 x i8> [[WIDE_LOAD_25]] to <16 x i16>
+; SSE-NEXT: [[TMP150:%.*]] = zext <16 x i8> [[WIDE_LOAD2_25]] to <16 x i16>
+; SSE-NEXT: [[TMP151:%.*]] = add <16 x i16> [[TMP145]], [[TMP149]]
+; SSE-NEXT: [[TMP152:%.*]] = add <16 x i16> [[TMP146]], [[TMP150]]
+; SSE-NEXT: [[TMP153:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 832
+; SSE-NEXT: [[TMP154:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 848
+; SSE-NEXT: [[WIDE_LOAD_26:%.*]] = load <16 x i8>, ptr [[TMP153]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_26:%.*]] = load <16 x i8>, ptr [[TMP154]], align 1
+; SSE-NEXT: [[TMP155:%.*]] = zext <16 x i8> [[WIDE_LOAD_26]] to <16 x i16>
+; SSE-NEXT: [[TMP156:%.*]] = zext <16 x i8> [[WIDE_LOAD2_26]] to <16 x i16>
+; SSE-NEXT: [[TMP157:%.*]] = add <16 x i16> [[TMP151]], [[TMP155]]
+; SSE-NEXT: [[TMP158:%.*]] = add <16 x i16> [[TMP152]], [[TMP156]]
+; SSE-NEXT: [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 864
+; SSE-NEXT: [[TMP160:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 880
+; SSE-NEXT: [[WIDE_LOAD_27:%.*]] = load <16 x i8>, ptr [[TMP159]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_27:%.*]] = load <16 x i8>, ptr [[TMP160]], align 1
+; SSE-NEXT: [[TMP161:%.*]] = zext <16 x i8> [[WIDE_LOAD_27]] to <16 x i16>
+; SSE-NEXT: [[TMP162:%.*]] = zext <16 x i8> [[WIDE_LOAD2_27]] to <16 x i16>
+; SSE-NEXT: [[TMP163:%.*]] = add <16 x i16> [[TMP157]], [[TMP161]]
+; SSE-NEXT: [[TMP164:%.*]] = add <16 x i16> [[TMP158]], [[TMP162]]
+; SSE-NEXT: [[TMP165:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 896
+; SSE-NEXT: [[TMP166:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 912
+; SSE-NEXT: [[WIDE_LOAD_28:%.*]] = load <16 x i8>, ptr [[TMP165]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_28:%.*]] = load <16 x i8>, ptr [[TMP166]], align 1
+; SSE-NEXT: [[TMP167:%.*]] = zext <16 x i8> [[WIDE_LOAD_28]] to <16 x i16>
+; SSE-NEXT: [[TMP168:%.*]] = zext <16 x i8> [[WIDE_LOAD2_28]] to <16 x i16>
+; SSE-NEXT: [[TMP169:%.*]] = add <16 x i16> [[TMP163]], [[TMP167]]
+; SSE-NEXT: [[TMP170:%.*]] = add <16 x i16> [[TMP164]], [[TMP168]]
+; SSE-NEXT: [[TMP171:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 928
+; SSE-NEXT: [[TMP172:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 944
+; SSE-NEXT: [[WIDE_LOAD_29:%.*]] = load <16 x i8>, ptr [[TMP171]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_29:%.*]] = load <16 x i8>, ptr [[TMP172]], align 1
+; SSE-NEXT: [[TMP173:%.*]] = zext <16 x i8> [[WIDE_LOAD_29]] to <16 x i16>
+; SSE-NEXT: [[TMP174:%.*]] = zext <16 x i8> [[WIDE_LOAD2_29]] to <16 x i16>
+; SSE-NEXT: [[TMP175:%.*]] = add <16 x i16> [[TMP169]], [[TMP173]]
+; SSE-NEXT: [[TMP176:%.*]] = add <16 x i16> [[TMP170]], [[TMP174]]
+; SSE-NEXT: [[TMP177:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 960
+; SSE-NEXT: [[TMP178:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 976
+; SSE-NEXT: [[WIDE_LOAD_30:%.*]] = load <16 x i8>, ptr [[TMP177]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_30:%.*]] = load <16 x i8>, ptr [[TMP178]], align 1
+; SSE-NEXT: [[TMP179:%.*]] = zext <16 x i8> [[WIDE_LOAD_30]] to <16 x i16>
+; SSE-NEXT: [[TMP180:%.*]] = zext <16 x i8> [[WIDE_LOAD2_30]] to <16 x i16>
+; SSE-NEXT: [[TMP181:%.*]] = add <16 x i16> [[TMP175]], [[TMP179]]
+; SSE-NEXT: [[TMP182:%.*]] = add <16 x i16> [[TMP176]], [[TMP180]]
+; SSE-NEXT: [[TMP183:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 992
+; SSE-NEXT: [[TMP184:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 1008
+; SSE-NEXT: [[WIDE_LOAD_31:%.*]] = load <16 x i8>, ptr [[TMP183]], align 1
+; SSE-NEXT: [[WIDE_LOAD2_31:%.*]] = load <16 x i8>, ptr [[TMP184]], align 1
+; SSE-NEXT: [[TMP185:%.*]] = zext <16 x i8> [[WIDE_LOAD_31]] to <16 x i16>
+; SSE-NEXT: [[TMP186:%.*]] = zext <16 x i8> [[WIDE_LOAD2_31]] to <16 x i16>
+; SSE-NEXT: [[TMP187:%.*]] = add <16 x i16> [[TMP181]], [[TMP185]]
+; SSE-NEXT: [[TMP188:%.*]] = add <16 x i16> [[TMP182]], [[TMP186]]
+; SSE-NEXT: [[BIN_RDX:%.*]] = add <16 x i16> [[TMP188]], [[TMP187]]
+; SSE-NEXT: [[TMP14:%.*]] = tail call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[BIN_RDX]])
; SSE-NEXT: ret i16 [[TMP14]]
;
; AVX-LABEL: @test(
; AVX-NEXT: entry:
; AVX-NEXT: [[FIRST:%.*]] = load i8, ptr [[PTR:%.*]], align 1
; AVX-NEXT: tail call void @use(i8 [[FIRST]]) #[[ATTR3:[0-9]+]]
-; AVX-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 16
; AVX-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 32
-; AVX-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 48
-; AVX-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[PTR]], align 1
-; AVX-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
-; AVX-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
-; AVX-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; AVX-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i16>
-; AVX-NEXT: [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD4]] to <16 x i16>
-; AVX-NEXT: [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD5]] to <16 x i16>
-; AVX-NEXT: [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD6]] to <16 x i16>
; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 64
-; AVX-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 80
; AVX-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 96
-; AVX-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 112
-; AVX-NEXT: [[WIDE_LOAD_1:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_1:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_1:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_1:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1
-; AVX-NEXT: [[TMP11:%.*]] = zext <16 x i8> [[WIDE_LOAD_1]] to <16 x i16>
-; AVX-NEXT: [[TMP12:%.*]] = zext <16 x i8> [[WIDE_LOAD4_1]] to <16 x i16>
-; AVX-NEXT: [[TMP13:%.*]] = zext <16 x i8> [[WIDE_LOAD5_1]] to <16 x i16>
-; AVX-NEXT: [[TMP14:%.*]] = zext <16 x i8> [[WIDE_LOAD6_1]] to <16 x i16>
-; AVX-NEXT: [[TMP15:%.*]] = add nuw nsw <16 x i16> [[TMP3]], [[TMP11]]
-; AVX-NEXT: [[TMP16:%.*]] = add nuw nsw <16 x i16> [[TMP4]], [[TMP12]]
-; AVX-NEXT: [[TMP17:%.*]] = add nuw nsw <16 x i16> [[TMP5]], [[TMP13]]
-; AVX-NEXT: [[TMP18:%.*]] = add nuw nsw <16 x i16> [[TMP6]], [[TMP14]]
+; AVX-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[PTR]], align 1
+; AVX-NEXT: [[WIDE_LOAD4:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
+; AVX-NEXT: [[WIDE_LOAD5:%.*]] = load <32 x i8>, ptr [[TMP7]], align 1
+; AVX-NEXT: [[WIDE_LOAD6:%.*]] = load <32 x i8>, ptr [[TMP9]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <32 x i8> [[WIDE_LOAD]] to <32 x i16>
+; AVX-NEXT: [[TMP4:%.*]] = zext <32 x i8> [[WIDE_LOAD4]] to <32 x i16>
+; AVX-NEXT: [[TMP5:%.*]] = zext <32 x i8> [[WIDE_LOAD5]] to <32 x i16>
+; AVX-NEXT: [[TMP6:%.*]] = zext <32 x i8> [[WIDE_LOAD6]] to <32 x i16>
; AVX-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 128
-; AVX-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 144
; AVX-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 160
-; AVX-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 176
-; AVX-NEXT: [[WIDE_LOAD_2:%.*]] = load <16 x i8>, ptr [[TMP19]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_2:%.*]] = load <16 x i8>, ptr [[TMP20]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_2:%.*]] = load <16 x i8>, ptr [[TMP21]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_2:%.*]] = load <16 x i8>, ptr [[TMP22]], align 1
-; AVX-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD_2]] to <16 x i16>
-; AVX-NEXT: [[TMP24:%.*]] = zext <16 x i8> [[WIDE_LOAD4_2]] to <16 x i16>
-; AVX-NEXT: [[TMP25:%.*]] = zext <16 x i8> [[WIDE_LOAD5_2]] to <16 x i16>
-; AVX-NEXT: [[TMP26:%.*]] = zext <16 x i8> [[WIDE_LOAD6_2]] to <16 x i16>
-; AVX-NEXT: [[TMP27:%.*]] = add nuw nsw <16 x i16> [[TMP15]], [[TMP23]]
-; AVX-NEXT: [[TMP28:%.*]] = add nuw nsw <16 x i16> [[TMP16]], [[TMP24]]
-; AVX-NEXT: [[TMP29:%.*]] = add nuw nsw <16 x i16> [[TMP17]], [[TMP25]]
-; AVX-NEXT: [[TMP30:%.*]] = add nuw nsw <16 x i16> [[TMP18]], [[TMP26]]
; AVX-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 192
-; AVX-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 208
; AVX-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 224
-; AVX-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 240
-; AVX-NEXT: [[WIDE_LOAD_3:%.*]] = load <16 x i8>, ptr [[TMP31]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_3:%.*]] = load <16 x i8>, ptr [[TMP32]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_3:%.*]] = load <16 x i8>, ptr [[TMP33]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_3:%.*]] = load <16 x i8>, ptr [[TMP34]], align 1
-; AVX-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD_3]] to <16 x i16>
-; AVX-NEXT: [[TMP36:%.*]] = zext <16 x i8> [[WIDE_LOAD4_3]] to <16 x i16>
-; AVX-NEXT: [[TMP37:%.*]] = zext <16 x i8> [[WIDE_LOAD5_3]] to <16 x i16>
-; AVX-NEXT: [[TMP38:%.*]] = zext <16 x i8> [[WIDE_LOAD6_3]] to <16 x i16>
-; AVX-NEXT: [[TMP39:%.*]] = add nuw nsw <16 x i16> [[TMP27]], [[TMP35]]
-; AVX-NEXT: [[TMP40:%.*]] = add nuw nsw <16 x i16> [[TMP28]], [[TMP36]]
-; AVX-NEXT: [[TMP41:%.*]] = add nuw nsw <16 x i16> [[TMP29]], [[TMP37]]
-; AVX-NEXT: [[TMP42:%.*]] = add nuw nsw <16 x i16> [[TMP30]], [[TMP38]]
+; AVX-NEXT: [[WIDE_LOAD_1:%.*]] = load <32 x i8>, ptr [[TMP19]], align 1
+; AVX-NEXT: [[WIDE_LOAD4_1:%.*]] = load <32 x i8>, ptr [[TMP21]], align 1
+; AVX-NEXT: [[WIDE_LOAD5_1:%.*]] = load <32 x i8>, ptr [[TMP31]], align 1
+; AVX-NEXT: [[WIDE_LOAD6_1:%.*]] = load <32 x i8>, ptr [[TMP33]], align 1
+; AVX-NEXT: [[TMP11:%.*]] = zext <32 x i8> [[WIDE_LOAD_1]] to <32 x i16>
+; AVX-NEXT: [[TMP12:%.*]] = zext <32 x i8> [[WIDE_LOAD4_1]] to <32 x i16>
+; AVX-NEXT: [[TMP13:%.*]] = zext <32 x i8> [[WIDE_LOAD5_1]] to <32 x i16>
+; AVX-NEXT: [[TMP14:%.*]] = zext <32 x i8> [[WIDE_LOAD6_1]] to <32 x i16>
+; AVX-NEXT: [[TMP15:%.*]] = add nuw nsw <32 x i16> [[TMP3]], [[TMP11]]
+; AVX-NEXT: [[TMP16:%.*]] = add nuw nsw <32 x i16> [[TMP4]], [[TMP12]]
+; AVX-NEXT: [[TMP17:%.*]] = add nuw nsw <32 x i16> [[TMP5]], [[TMP13]]
+; AVX-NEXT: [[TMP18:%.*]] = add nuw nsw <32 x i16> [[TMP6]], [[TMP14]]
; AVX-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 256
-; AVX-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 272
; AVX-NEXT: [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 288
-; AVX-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 304
-; AVX-NEXT: [[WIDE_LOAD_4:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_4:%.*]] = load <16 x i8>, ptr [[TMP44]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_4:%.*]] = load <16 x i8>, ptr [[TMP45]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_4:%.*]] = load <16 x i8>, ptr [[TMP46]], align 1
-; AVX-NEXT: [[TMP47:%.*]] = zext <16 x i8> [[WIDE_LOAD_4]] to <16 x i16>
-; AVX-NEXT: [[TMP48:%.*]] = zext <16 x i8> [[WIDE_LOAD4_4]] to <16 x i16>
-; AVX-NEXT: [[TMP49:%.*]] = zext <16 x i8> [[WIDE_LOAD5_4]] to <16 x i16>
-; AVX-NEXT: [[TMP50:%.*]] = zext <16 x i8> [[WIDE_LOAD6_4]] to <16 x i16>
-; AVX-NEXT: [[TMP51:%.*]] = add nuw nsw <16 x i16> [[TMP39]], [[TMP47]]
-; AVX-NEXT: [[TMP52:%.*]] = add nuw nsw <16 x i16> [[TMP40]], [[TMP48]]
-; AVX-NEXT: [[TMP53:%.*]] = add nuw nsw <16 x i16> [[TMP41]], [[TMP49]]
-; AVX-NEXT: [[TMP54:%.*]] = add nuw nsw <16 x i16> [[TMP42]], [[TMP50]]
; AVX-NEXT: [[TMP55:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 320
-; AVX-NEXT: [[TMP56:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
; AVX-NEXT: [[TMP57:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 352
-; AVX-NEXT: [[TMP58:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 368
-; AVX-NEXT: [[WIDE_LOAD_5:%.*]] = load <16 x i8>, ptr [[TMP55]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_5:%.*]] = load <16 x i8>, ptr [[TMP56]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_5:%.*]] = load <16 x i8>, ptr [[TMP57]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_5:%.*]] = load <16 x i8>, ptr [[TMP58]], align 1
-; AVX-NEXT: [[TMP59:%.*]] = zext <16 x i8> [[WIDE_LOAD_5]] to <16 x i16>
-; AVX-NEXT: [[TMP60:%.*]] = zext <16 x i8> [[WIDE_LOAD4_5]] to <16 x i16>
-; AVX-NEXT: [[TMP61:%.*]] = zext <16 x i8> [[WIDE_LOAD5_5]] to <16 x i16>
-; AVX-NEXT: [[TMP62:%.*]] = zext <16 x i8> [[WIDE_LOAD6_5]] to <16 x i16>
-; AVX-NEXT: [[TMP63:%.*]] = add nuw nsw <16 x i16> [[TMP51]], [[TMP59]]
-; AVX-NEXT: [[TMP64:%.*]] = add nuw nsw <16 x i16> [[TMP52]], [[TMP60]]
-; AVX-NEXT: [[TMP65:%.*]] = add nuw nsw <16 x i16> [[TMP53]], [[TMP61]]
-; AVX-NEXT: [[TMP66:%.*]] = add nuw nsw <16 x i16> [[TMP54]], [[TMP62]]
+; AVX-NEXT: [[WIDE_LOAD_2:%.*]] = load <32 x i8>, ptr [[TMP43]], align 1
+; AVX-NEXT: [[WIDE_LOAD4_2:%.*]] = load <32 x i8>, ptr [[TMP45]], align 1
+; AVX-NEXT: [[WIDE_LOAD5_2:%.*]] = load <32 x i8>, ptr [[TMP55]], align 1
+; AVX-NEXT: [[WIDE_LOAD6_2:%.*]] = load <32 x i8>, ptr [[TMP57]], align 1
+; AVX-NEXT: [[TMP23:%.*]] = zext <32 x i8> [[WIDE_LOAD_2]] to <32 x i16>
+; AVX-NEXT: [[TMP24:%.*]] = zext <32 x i8> [[WIDE_LOAD4_2]] to <32 x i16>
+; AVX-NEXT: [[TMP25:%.*]] = zext <32 x i8> [[WIDE_LOAD5_2]] to <32 x i16>
+; AVX-NEXT: [[TMP26:%.*]] = zext <32 x i8> [[WIDE_LOAD6_2]] to <32 x i16>
+; AVX-NEXT: [[TMP27:%.*]] = add nuw nsw <32 x i16> [[TMP15]], [[TMP23]]
+; AVX-NEXT: [[TMP28:%.*]] = add nuw nsw <32 x i16> [[TMP16]], [[TMP24]]
+; AVX-NEXT: [[TMP29:%.*]] = add nuw nsw <32 x i16> [[TMP17]], [[TMP25]]
+; AVX-NEXT: [[TMP30:%.*]] = add nuw nsw <32 x i16> [[TMP18]], [[TMP26]]
; AVX-NEXT: [[TMP67:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 384
-; AVX-NEXT: [[TMP68:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 400
; AVX-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 416
-; AVX-NEXT: [[TMP70:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 432
-; AVX-NEXT: [[WIDE_LOAD_6:%.*]] = load <16 x i8>, ptr [[TMP67]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_6:%.*]] = load <16 x i8>, ptr [[TMP68]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_6:%.*]] = load <16 x i8>, ptr [[TMP69]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_6:%.*]] = load <16 x i8>, ptr [[TMP70]], align 1
-; AVX-NEXT: [[TMP71:%.*]] = zext <16 x i8> [[WIDE_LOAD_6]] to <16 x i16>
-; AVX-NEXT: [[TMP72:%.*]] = zext <16 x i8> [[WIDE_LOAD4_6]] to <16 x i16>
-; AVX-NEXT: [[TMP73:%.*]] = zext <16 x i8> [[WIDE_LOAD5_6]] to <16 x i16>
-; AVX-NEXT: [[TMP74:%.*]] = zext <16 x i8> [[WIDE_LOAD6_6]] to <16 x i16>
-; AVX-NEXT: [[TMP75:%.*]] = add nuw nsw <16 x i16> [[TMP63]], [[TMP71]]
-; AVX-NEXT: [[TMP76:%.*]] = add nuw nsw <16 x i16> [[TMP64]], [[TMP72]]
-; AVX-NEXT: [[TMP77:%.*]] = add nuw nsw <16 x i16> [[TMP65]], [[TMP73]]
-; AVX-NEXT: [[TMP78:%.*]] = add nuw nsw <16 x i16> [[TMP66]], [[TMP74]]
; AVX-NEXT: [[TMP79:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 448
-; AVX-NEXT: [[TMP80:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 464
; AVX-NEXT: [[TMP81:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 480
-; AVX-NEXT: [[TMP82:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 496
-; AVX-NEXT: [[WIDE_LOAD_7:%.*]] = load <16 x i8>, ptr [[TMP79]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_7:%.*]] = load <16 x i8>, ptr [[TMP80]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_7:%.*]] = load <16 x i8>, ptr [[TMP81]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_7:%.*]] = load <16 x i8>, ptr [[TMP82]], align 1
-; AVX-NEXT: [[TMP83:%.*]] = zext <16 x i8> [[WIDE_LOAD_7]] to <16 x i16>
-; AVX-NEXT: [[TMP84:%.*]] = zext <16 x i8> [[WIDE_LOAD4_7]] to <16 x i16>
-; AVX-NEXT: [[TMP85:%.*]] = zext <16 x i8> [[WIDE_LOAD5_7]] to <16 x i16>
-; AVX-NEXT: [[TMP86:%.*]] = zext <16 x i8> [[WIDE_LOAD6_7]] to <16 x i16>
-; AVX-NEXT: [[TMP87:%.*]] = add <16 x i16> [[TMP75]], [[TMP83]]
-; AVX-NEXT: [[TMP88:%.*]] = add <16 x i16> [[TMP76]], [[TMP84]]
-; AVX-NEXT: [[TMP89:%.*]] = add <16 x i16> [[TMP77]], [[TMP85]]
-; AVX-NEXT: [[TMP90:%.*]] = add <16 x i16> [[TMP78]], [[TMP86]]
+; AVX-NEXT: [[WIDE_LOAD_3:%.*]] = load <32 x i8>, ptr [[TMP67]], align 1
+; AVX-NEXT: [[WIDE_LOAD4_3:%.*]] = load <32 x i8>, ptr [[TMP69]], align 1
+; AVX-NEXT: [[WIDE_LOAD5_3:%.*]] = load <32 x i8>, ptr [[TMP79]], align 1
+; AVX-NEXT: [[WIDE_LOAD6_3:%.*]] = load <32 x i8>, ptr [[TMP81]], align 1
+; AVX-NEXT: [[TMP35:%.*]] = zext <32 x i8> [[WIDE_LOAD_3]] to <32 x i16>
+; AVX-NEXT: [[TMP36:%.*]] = zext <32 x i8> [[WIDE_LOAD4_3]] to <32 x i16>
+; AVX-NEXT: [[TMP37:%.*]] = zext <32 x i8> [[WIDE_LOAD5_3]] to <32 x i16>
+; AVX-NEXT: [[TMP38:%.*]] = zext <32 x i8> [[WIDE_LOAD6_3]] to <32 x i16>
+; AVX-NEXT: [[TMP39:%.*]] = add nuw nsw <32 x i16> [[TMP27]], [[TMP35]]
+; AVX-NEXT: [[TMP40:%.*]] = add nuw nsw <32 x i16> [[TMP28]], [[TMP36]]
+; AVX-NEXT: [[TMP41:%.*]] = add nuw nsw <32 x i16> [[TMP29]], [[TMP37]]
+; AVX-NEXT: [[TMP42:%.*]] = add nuw nsw <32 x i16> [[TMP30]], [[TMP38]]
; AVX-NEXT: [[TMP91:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 512
-; AVX-NEXT: [[TMP92:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 528
; AVX-NEXT: [[TMP93:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 544
-; AVX-NEXT: [[TMP94:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 560
-; AVX-NEXT: [[WIDE_LOAD_8:%.*]] = load <16 x i8>, ptr [[TMP91]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_8:%.*]] = load <16 x i8>, ptr [[TMP92]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_8:%.*]] = load <16 x i8>, ptr [[TMP93]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_8:%.*]] = load <16 x i8>, ptr [[TMP94]], align 1
-; AVX-NEXT: [[TMP95:%.*]] = zext <16 x i8> [[WIDE_LOAD_8]] to <16 x i16>
-; AVX-NEXT: [[TMP96:%.*]] = zext <16 x i8> [[WIDE_LOAD4_8]] to <16 x i16>
-; AVX-NEXT: [[TMP97:%.*]] = zext <16 x i8> [[WIDE_LOAD5_8]] to <16 x i16>
-; AVX-NEXT: [[TMP98:%.*]] = zext <16 x i8> [[WIDE_LOAD6_8]] to <16 x i16>
-; AVX-NEXT: [[TMP99:%.*]] = add <16 x i16> [[TMP87]], [[TMP95]]
-; AVX-NEXT: [[TMP100:%.*]] = add <16 x i16> [[TMP88]], [[TMP96]]
-; AVX-NEXT: [[TMP101:%.*]] = add <16 x i16> [[TMP89]], [[TMP97]]
-; AVX-NEXT: [[TMP102:%.*]] = add <16 x i16> [[TMP90]], [[TMP98]]
; AVX-NEXT: [[TMP103:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 576
-; AVX-NEXT: [[TMP104:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 592
; AVX-NEXT: [[TMP105:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 608
-; AVX-NEXT: [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 624
-; AVX-NEXT: [[WIDE_LOAD_9:%.*]] = load <16 x i8>, ptr [[TMP103]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_9:%.*]] = load <16 x i8>, ptr [[TMP104]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_9:%.*]] = load <16 x i8>, ptr [[TMP105]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_9:%.*]] = load <16 x i8>, ptr [[TMP106]], align 1
-; AVX-NEXT: [[TMP107:%.*]] = zext <16 x i8> [[WIDE_LOAD_9]] to <16 x i16>
-; AVX-NEXT: [[TMP108:%.*]] = zext <16 x i8> [[WIDE_LOAD4_9]] to <16 x i16>
-; AVX-NEXT: [[TMP109:%.*]] = zext <16 x i8> [[WIDE_LOAD5_9]] to <16 x i16>
-; AVX-NEXT: [[TMP110:%.*]] = zext <16 x i8> [[WIDE_LOAD6_9]] to <16 x i16>
-; AVX-NEXT: [[TMP111:%.*]] = add <16 x i16> [[TMP99]], [[TMP107]]
-; AVX-NEXT: [[TMP112:%.*]] = add <16 x i16> [[TMP100]], [[TMP108]]
-; AVX-NEXT: [[TMP113:%.*]] = add <16 x i16> [[TMP101]], [[TMP109]]
-; AVX-NEXT: [[TMP114:%.*]] = add <16 x i16> [[TMP102]], [[TMP110]]
+; AVX-NEXT: [[WIDE_LOAD_4:%.*]] = load <32 x i8>, ptr [[TMP91]], align 1
+; AVX-NEXT: [[WIDE_LOAD4_4:%.*]] = load <32 x i8>, ptr [[TMP93]], align 1
+; AVX-NEXT: [[WIDE_LOAD5_4:%.*]] = load <32 x i8>, ptr [[TMP103]], align 1
+; AVX-NEXT: [[WIDE_LOAD6_4:%.*]] = load <32 x i8>, ptr [[TMP105]], align 1
+; AVX-NEXT: [[TMP47:%.*]] = zext <32 x i8> [[WIDE_LOAD_4]] to <32 x i16>
+; AVX-NEXT: [[TMP48:%.*]] = zext <32 x i8> [[WIDE_LOAD4_4]] to <32 x i16>
+; AVX-NEXT: [[TMP49:%.*]] = zext <32 x i8> [[WIDE_LOAD5_4]] to <32 x i16>
+; AVX-NEXT: [[TMP50:%.*]] = zext <32 x i8> [[WIDE_LOAD6_4]] to <32 x i16>
+; AVX-NEXT: [[TMP51:%.*]] = add nuw nsw <32 x i16> [[TMP39]], [[TMP47]]
+; AVX-NEXT: [[TMP52:%.*]] = add nuw nsw <32 x i16> [[TMP40]], [[TMP48]]
+; AVX-NEXT: [[TMP53:%.*]] = add nuw nsw <32 x i16> [[TMP41]], [[TMP49]]
+; AVX-NEXT: [[TMP54:%.*]] = add nuw nsw <32 x i16> [[TMP42]], [[TMP50]]
; AVX-NEXT: [[TMP115:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 640
-; AVX-NEXT: [[TMP116:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 656
; AVX-NEXT: [[TMP117:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 672
-; AVX-NEXT: [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 688
-; AVX-NEXT: [[WIDE_LOAD_10:%.*]] = load <16 x i8>, ptr [[TMP115]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_10:%.*]] = load <16 x i8>, ptr [[TMP116]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_10:%.*]] = load <16 x i8>, ptr [[TMP117]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_10:%.*]] = load <16 x i8>, ptr [[TMP118]], align 1
-; AVX-NEXT: [[TMP119:%.*]] = zext <16 x i8> [[WIDE_LOAD_10]] to <16 x i16>
-; AVX-NEXT: [[TMP120:%.*]] = zext <16 x i8> [[WIDE_LOAD4_10]] to <16 x i16>
-; AVX-NEXT: [[TMP121:%.*]] = zext <16 x i8> [[WIDE_LOAD5_10]] to <16 x i16>
-; AVX-NEXT: [[TMP122:%.*]] = zext <16 x i8> [[WIDE_LOAD6_10]] to <16 x i16>
-; AVX-NEXT: [[TMP123:%.*]] = add <16 x i16> [[TMP111]], [[TMP119]]
-; AVX-NEXT: [[TMP124:%.*]] = add <16 x i16> [[TMP112]], [[TMP120]]
-; AVX-NEXT: [[TMP125:%.*]] = add <16 x i16> [[TMP113]], [[TMP121]]
-; AVX-NEXT: [[TMP126:%.*]] = add <16 x i16> [[TMP114]], [[TMP122]]
; AVX-NEXT: [[TMP127:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 704
-; AVX-NEXT: [[TMP128:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 720
; AVX-NEXT: [[TMP129:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 736
-; AVX-NEXT: [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 752
-; AVX-NEXT: [[WIDE_LOAD_11:%.*]] = load <16 x i8>, ptr [[TMP127]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_11:%.*]] = load <16 x i8>, ptr [[TMP128]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_11:%.*]] = load <16 x i8>, ptr [[TMP129]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_11:%.*]] = load <16 x i8>, ptr [[TMP130]], align 1
-; AVX-NEXT: [[TMP131:%.*]] = zext <16 x i8> [[WIDE_LOAD_11]] to <16 x i16>
-; AVX-NEXT: [[TMP132:%.*]] = zext <16 x i8> [[WIDE_LOAD4_11]] to <16 x i16>
-; AVX-NEXT: [[TMP133:%.*]] = zext <16 x i8> [[WIDE_LOAD5_11]] to <16 x i16>
-; AVX-NEXT: [[TMP134:%.*]] = zext <16 x i8> [[WIDE_LOAD6_11]] to <16 x i16>
-; AVX-NEXT: [[TMP135:%.*]] = add <16 x i16> [[TMP123]], [[TMP131]]
-; AVX-NEXT: [[TMP136:%.*]] = add <16 x i16> [[TMP124]], [[TMP132]]
-; AVX-NEXT: [[TMP137:%.*]] = add <16 x i16> [[TMP125]], [[TMP133]]
-; AVX-NEXT: [[TMP138:%.*]] = add <16 x i16> [[TMP126]], [[TMP134]]
+; AVX-NEXT: [[WIDE_LOAD_5:%.*]] = load <32 x i8>, ptr [[TMP115]], align 1
+; AVX-NEXT: [[WIDE_LOAD4_5:%.*]] = load <32 x i8>, ptr [[TMP117]], align 1
+; AVX-NEXT: [[WIDE_LOAD5_5:%.*]] = load <32 x i8>, ptr [[TMP127]], align 1
+; AVX-NEXT: [[WIDE_LOAD6_5:%.*]] = load <32 x i8>, ptr [[TMP129]], align 1
+; AVX-NEXT: [[TMP59:%.*]] = zext <32 x i8> [[WIDE_LOAD_5]] to <32 x i16>
+; AVX-NEXT: [[TMP60:%.*]] = zext <32 x i8> [[WIDE_LOAD4_5]] to <32 x i16>
+; AVX-NEXT: [[TMP61:%.*]] = zext <32 x i8> [[WIDE_LOAD5_5]] to <32 x i16>
+; AVX-NEXT: [[TMP62:%.*]] = zext <32 x i8> [[WIDE_LOAD6_5]] to <32 x i16>
+; AVX-NEXT: [[TMP63:%.*]] = add nuw nsw <32 x i16> [[TMP51]], [[TMP59]]
+; AVX-NEXT: [[TMP64:%.*]] = add nuw nsw <32 x i16> [[TMP52]], [[TMP60]]
+; AVX-NEXT: [[TMP65:%.*]] = add nuw nsw <32 x i16> [[TMP53]], [[TMP61]]
+; AVX-NEXT: [[TMP66:%.*]] = add nuw nsw <32 x i16> [[TMP54]], [[TMP62]]
; AVX-NEXT: [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 768
-; AVX-NEXT: [[TMP140:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 784
; AVX-NEXT: [[TMP141:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 800
-; AVX-NEXT: [[TMP142:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 816
-; AVX-NEXT: [[WIDE_LOAD_12:%.*]] = load <16 x i8>, ptr [[TMP139]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_12:%.*]] = load <16 x i8>, ptr [[TMP140]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_12:%.*]] = load <16 x i8>, ptr [[TMP141]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_12:%.*]] = load <16 x i8>, ptr [[TMP142]], align 1
-; AVX-NEXT: [[TMP143:%.*]] = zext <16 x i8> [[WIDE_LOAD_12]] to <16 x i16>
-; AVX-NEXT: [[TMP144:%.*]] = zext <16 x i8> [[WIDE_LOAD4_12]] to <16 x i16>
-; AVX-NEXT: [[TMP145:%.*]] = zext <16 x i8> [[WIDE_LOAD5_12]] to <16 x i16>
-; AVX-NEXT: [[TMP146:%.*]] = zext <16 x i8> [[WIDE_LOAD6_12]] to <16 x i16>
-; AVX-NEXT: [[TMP147:%.*]] = add <16 x i16> [[TMP135]], [[TMP143]]
-; AVX-NEXT: [[TMP148:%.*]] = add <16 x i16> [[TMP136]], [[TMP144]]
-; AVX-NEXT: [[TMP149:%.*]] = add <16 x i16> [[TMP137]], [[TMP145]]
-; AVX-NEXT: [[TMP150:%.*]] = add <16 x i16> [[TMP138]], [[TMP146]]
; AVX-NEXT: [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 832
-; AVX-NEXT: [[TMP152:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 848
; AVX-NEXT: [[TMP153:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 864
-; AVX-NEXT: [[TMP154:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 880
-; AVX-NEXT: [[WIDE_LOAD_13:%.*]] = load <16 x i8>, ptr [[TMP151]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_13:%.*]] = load <16 x i8>, ptr [[TMP152]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_13:%.*]] = load <16 x i8>, ptr [[TMP153]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_13:%.*]] = load <16 x i8>, ptr [[TMP154]], align 1
-; AVX-NEXT: [[TMP155:%.*]] = zext <16 x i8> [[WIDE_LOAD_13]] to <16 x i16>
-; AVX-NEXT: [[TMP156:%.*]] = zext <16 x i8> [[WIDE_LOAD4_13]] to <16 x i16>
-; AVX-NEXT: [[TMP157:%.*]] = zext <16 x i8> [[WIDE_LOAD5_13]] to <16 x i16>
-; AVX-NEXT: [[TMP158:%.*]] = zext <16 x i8> [[WIDE_LOAD6_13]] to <16 x i16>
-; AVX-NEXT: [[TMP159:%.*]] = add <16 x i16> [[TMP147]], [[TMP155]]
-; AVX-NEXT: [[TMP160:%.*]] = add <16 x i16> [[TMP148]], [[TMP156]]
-; AVX-NEXT: [[TMP161:%.*]] = add <16 x i16> [[TMP149]], [[TMP157]]
-; AVX-NEXT: [[TMP162:%.*]] = add <16 x i16> [[TMP150]], [[TMP158]]
+; AVX-NEXT: [[WIDE_LOAD_6:%.*]] = load <32 x i8>, ptr [[TMP139]], align 1
+; AVX-NEXT: [[WIDE_LOAD4_6:%.*]] = load <32 x i8>, ptr [[TMP141]], align 1
+; AVX-NEXT: [[WIDE_LOAD5_6:%.*]] = load <32 x i8>, ptr [[TMP151]], align 1
+; AVX-NEXT: [[WIDE_LOAD6_6:%.*]] = load <32 x i8>, ptr [[TMP153]], align 1
+; AVX-NEXT: [[TMP71:%.*]] = zext <32 x i8> [[WIDE_LOAD_6]] to <32 x i16>
+; AVX-NEXT: [[TMP72:%.*]] = zext <32 x i8> [[WIDE_LOAD4_6]] to <32 x i16>
+; AVX-NEXT: [[TMP73:%.*]] = zext <32 x i8> [[WIDE_LOAD5_6]] to <32 x i16>
+; AVX-NEXT: [[TMP74:%.*]] = zext <32 x i8> [[WIDE_LOAD6_6]] to <32 x i16>
+; AVX-NEXT: [[TMP75:%.*]] = add nuw nsw <32 x i16> [[TMP63]], [[TMP71]]
+; AVX-NEXT: [[TMP76:%.*]] = add nuw nsw <32 x i16> [[TMP64]], [[TMP72]]
+; AVX-NEXT: [[TMP77:%.*]] = add nuw nsw <32 x i16> [[TMP65]], [[TMP73]]
+; AVX-NEXT: [[TMP78:%.*]] = add nuw nsw <32 x i16> [[TMP66]], [[TMP74]]
; AVX-NEXT: [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 896
-; AVX-NEXT: [[TMP164:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 912
; AVX-NEXT: [[TMP165:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 928
-; AVX-NEXT: [[TMP166:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 944
-; AVX-NEXT: [[WIDE_LOAD_14:%.*]] = load <16 x i8>, ptr [[TMP163]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_14:%.*]] = load <16 x i8>, ptr [[TMP164]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_14:%.*]] = load <16 x i8>, ptr [[TMP165]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_14:%.*]] = load <16 x i8>, ptr [[TMP166]], align 1
-; AVX-NEXT: [[TMP167:%.*]] = zext <16 x i8> [[WIDE_LOAD_14]] to <16 x i16>
-; AVX-NEXT: [[TMP168:%.*]] = zext <16 x i8> [[WIDE_LOAD4_14]] to <16 x i16>
-; AVX-NEXT: [[TMP169:%.*]] = zext <16 x i8> [[WIDE_LOAD5_14]] to <16 x i16>
-; AVX-NEXT: [[TMP170:%.*]] = zext <16 x i8> [[WIDE_LOAD6_14]] to <16 x i16>
-; AVX-NEXT: [[TMP171:%.*]] = add <16 x i16> [[TMP159]], [[TMP167]]
-; AVX-NEXT: [[TMP172:%.*]] = add <16 x i16> [[TMP160]], [[TMP168]]
-; AVX-NEXT: [[TMP173:%.*]] = add <16 x i16> [[TMP161]], [[TMP169]]
-; AVX-NEXT: [[TMP174:%.*]] = add <16 x i16> [[TMP162]], [[TMP170]]
; AVX-NEXT: [[TMP175:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 960
-; AVX-NEXT: [[TMP176:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 976
; AVX-NEXT: [[TMP177:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 992
-; AVX-NEXT: [[TMP178:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 1008
-; AVX-NEXT: [[WIDE_LOAD_15:%.*]] = load <16 x i8>, ptr [[TMP175]], align 1
-; AVX-NEXT: [[WIDE_LOAD4_15:%.*]] = load <16 x i8>, ptr [[TMP176]], align 1
-; AVX-NEXT: [[WIDE_LOAD5_15:%.*]] = load <16 x i8>, ptr [[TMP177]], align 1
-; AVX-NEXT: [[WIDE_LOAD6_15:%.*]] = load <16 x i8>, ptr [[TMP178]], align 1
-; AVX-NEXT: [[TMP179:%.*]] = zext <16 x i8> [[WIDE_LOAD_15]] to <16 x i16>
-; AVX-NEXT: [[TMP180:%.*]] = zext <16 x i8> [[WIDE_LOAD4_15]] to <16 x i16>
-; AVX-NEXT: [[TMP181:%.*]] = zext <16 x i8> [[WIDE_LOAD5_15]] to <16 x i16>
-; AVX-NEXT: [[TMP182:%.*]] = zext <16 x i8> [[WIDE_LOAD6_15]] to <16 x i16>
-; AVX-NEXT: [[TMP183:%.*]] = add <16 x i16> [[TMP171]], [[TMP179]]
-; AVX-NEXT: [[TMP184:%.*]] = add <16 x i16> [[TMP172]], [[TMP180]]
-; AVX-NEXT: [[TMP185:%.*]] = add <16 x i16> [[TMP173]], [[TMP181]]
-; AVX-NEXT: [[TMP186:%.*]] = add <16 x i16> [[TMP174]], [[TMP182]]
-; AVX-NEXT: [[BIN_RDX:%.*]] = add <16 x i16> [[TMP184]], [[TMP183]]
-; AVX-NEXT: [[BIN_RDX7:%.*]] = add <16 x i16> [[TMP185]], [[BIN_RDX]]
-; AVX-NEXT: [[BIN_RDX8:%.*]] = add <16 x i16> [[TMP186]], [[BIN_RDX7]]
-; AVX-NEXT: [[TMP187:%.*]] = tail call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[BIN_RDX8]])
+; AVX-NEXT: [[WIDE_LOAD_7:%.*]] = load <32 x i8>, ptr [[TMP163]], align 1
+; AVX-NEXT: [[WIDE_LOAD4_7:%.*]] = load <32 x i8>, ptr [[TMP165]], align 1
+; AVX-NEXT: [[WIDE_LOAD5_7:%.*]] = load <32 x i8>, ptr [[TMP175]], align 1
+; AVX-NEXT: [[WIDE_LOAD6_7:%.*]] = load <32 x i8>, ptr [[TMP177]], align 1
+; AVX-NEXT: [[TMP83:%.*]] = zext <32 x i8> [[WIDE_LOAD_7]] to <32 x i16>
+; AVX-NEXT: [[TMP84:%.*]] = zext <32 x i8> [[WIDE_LOAD4_7]] to <32 x i16>
+; AVX-NEXT: [[TMP85:%.*]] = zext <32 x i8> [[WIDE_LOAD5_7]] to <32 x i16>
+; AVX-NEXT: [[TMP86:%.*]] = zext <32 x i8> [[WIDE_LOAD6_7]] to <32 x i16>
+; AVX-NEXT: [[TMP87:%.*]] = add <32 x i16> [[TMP75]], [[TMP83]]
+; AVX-NEXT: [[TMP88:%.*]] = add <32 x i16> [[TMP76]], [[TMP84]]
+; AVX-NEXT: [[TMP89:%.*]] = add <32 x i16> [[TMP77]], [[TMP85]]
+; AVX-NEXT: [[TMP90:%.*]] = add <32 x i16> [[TMP78]], [[TMP86]]
+; AVX-NEXT: [[BIN_RDX:%.*]] = add <32 x i16> [[TMP88]], [[TMP87]]
+; AVX-NEXT: [[BIN_RDX7:%.*]] = add <32 x i16> [[TMP89]], [[BIN_RDX]]
+; AVX-NEXT: [[BIN_RDX8:%.*]] = add <32 x i16> [[TMP90]], [[BIN_RDX7]]
+; AVX-NEXT: [[TMP187:%.*]] = tail call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> [[BIN_RDX8]])
; AVX-NEXT: ret i16 [[TMP187]]
;
entry:
More information about the llvm-commits
mailing list