[llvm] [PowerPC] Add PPC cost model support for partial reductions (PR #214760)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 12:04:44 PDT 2026
https://github.com/RolandF77 updated https://github.com/llvm/llvm-project/pull/214760
>From 7b9dae1638f79e9b26573462408019c6def1f2ab Mon Sep 17 00:00:00 2001
From: Roland Froese <froese at ca.ibm.com>
Date: Fri, 31 Jul 2026 20:13:03 +0000
Subject: [PATCH 1/6] cost partial reductions
---
.../Target/PowerPC/PPCTargetTransformInfo.cpp | 38 +
.../Target/PowerPC/PPCTargetTransformInfo.h | 4 +-
.../LoopVectorize/PowerPC/cost-partial-red.ll | 157 ++
.../LoopVectorize/PowerPC/partial-red.ll | 2003 +++++++++++++++++
4 files changed, 2199 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/cost-partial-red.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index b3a54fd4c1ef4..ea4539e08c672 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -1160,3 +1160,41 @@ PPCTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
Cost += 1; // need shift for length
return Cost;
}
+
+InstructionCost PPCTTIImpl::getPartialReductionCost(
+ unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
+ ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
+ TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+ TTI::TargetCostKind CostKind,
+ std::optional<FastMathFlags> FMF) const {
+ InstructionCost Invalid = InstructionCost::getInvalid();
+
+ if (Opcode != Instruction::Add)
+ return Invalid;
+ if (BinOp && BinOp.value() != Instruction::Mul)
+ return Invalid;
+
+ EVT AccVT = TLI->getValueType(DL, AccumType, true);
+ if (AccVT != MVT::i32)
+ return Invalid;
+ if (InputTypeA != InputTypeB)
+ return Invalid;
+
+ Type *ATy = VectorType::get(InputTypeA, VF);
+ EVT AVT = TLI->getValueType(DL, ATy, true);
+ if (AVT != MVT::v16i8 && AVT != MVT::v8i16)
+ return Invalid;
+
+ // For v16i8 PPC has vmsumubm zext/zext and vmsummbm sext/zext
+ // For v8i16 PPC has vmsumuhm zext/zext and vmsumshm sext/sext
+ if (OpAExtend != TTI::PR_SignExtend && OpAExtend != TTI::PR_ZeroExtend)
+ return Invalid;
+ if (OpAExtend != OpBExtend) {
+ if (AVT != MVT::v16i8 || OpBExtend != TTI::PR_ZeroExtend)
+ return Invalid;
+ } else if (AVT == MVT::v16i8 && OpAExtend == TTI::PR_SignExtend) {
+ return Invalid;
+ }
+
+ return vectorCostAdjustmentFactor(Instruction::Add, ATy, nullptr);
+}
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index c8ec1960b17dd..c4673cd2fe856 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -173,9 +173,7 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
TTI::TargetCostKind CostKind,
- std::optional<FastMathFlags> FMF) const override {
- return InstructionCost::getInvalid();
- }
+ std::optional<FastMathFlags> FMF) const override;
private:
// The following constant is used for estimating costs on power9.
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/cost-partial-red.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/cost-partial-red.ll
new file mode 100644
index 0000000000000..e5e34da03dbef
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/cost-partial-red.ll
@@ -0,0 +1,157 @@
+; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr8 -S -passes=loop-vectorize -disable-output -debug-only=loop-vectorize < %s 2>&1 | FileCheck --check-prefix=P8COST %s
+; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr9 -S -passes=loop-vectorize -disable-output -debug-only=loop-vectorize < %s 2>&1 | FileCheck --check-prefix=P9COST %s
+; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr10 -S -passes=loop-vectorize -disable-output -debug-only=loop-vectorize < %s 2>&1 | FileCheck --check-prefix=P8COST %s
+
+target datalayout = "e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8COST,P9COST: LV: Checking a loop in 'dotu8'
+; P8COST: Cost of 1 for VF 16: EXPRESSION vp<%8> = ir<%sum.08> + partial.reduce.add (mul nuw nsw (ir<%1> zext to i32), (ir<%0> zext to i32))
+; P8COST: Cost for VF 16: 5 (Estimated cost per lane: 0.313)
+; P9COST: Cost of 2 for VF 16: EXPRESSION vp<%8> = ir<%sum.08> + partial.reduce.add (mul nuw nsw (ir<%1> zext to i32), (ir<%0> zext to i32))
+; P9COST: Cost for VF 16: 8 (Estimated cost per lane: 0.5)
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ %add.lcssa = phi i32 [ %add, %for.body ]
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.09
+ %0 = load i8, ptr %arrayidx, align 1, !tbaa !8
+ %conv = zext i8 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i64 %i.09
+ %1 = load i8, ptr %arrayidx1, align 1, !tbaa !8
+ %conv2 = zext i8 %1 to i32
+ %mul = mul nuw nsw i32 %conv2, %conv
+ %add = add nuw nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !9
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8COST,P9COST: LV: Checking a loop in 'dots8'
+; P8COST: Cost for VF 16: 26 (Estimated cost per lane: 1.63)
+; P9COST: Cost for VF 16: 36 (Estimated cost per lane: 2.25)
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %entry, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.09
+ %0 = load i8, ptr %arrayidx, align 1, !tbaa !8
+ %conv = sext i8 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i64 %i.09
+ %1 = load i8, ptr %arrayidx1, align 1, !tbaa !8
+ %conv2 = sext i8 %1 to i32
+ %mul = mul nsw i32 %conv2, %conv
+ %add = add nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !11
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8COST,P9COST: LV: Checking a loop in 'dotu16'
+; P8COST: Cost of 1 for VF 8: EXPRESSION vp<%8> = ir<%sum.08> + partial.reduce.add (mul nuw nsw (ir<%1> zext to i32), (ir<%0> zext to i32))
+; P8COST: Cost for VF 8: 5 (Estimated cost per lane: 0.625)
+; P9COST: Cost of 2 for VF 8: EXPRESSION vp<%8> = ir<%sum.08> + partial.reduce.add (mul nuw nsw (ir<%1> zext to i32), (ir<%0> zext to i32))
+; P9COST: Cost for VF 8: 8 (Estimated cost per lane: 1)
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %entry, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw [2 x i8], ptr %a, i64 %i.09
+ %0 = load i16, ptr %arrayidx, align 2, !tbaa !12
+ %conv = zext i16 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw [2 x i8], ptr %b, i64 %i.09
+ %1 = load i16, ptr %arrayidx1, align 2, !tbaa !12
+ %conv2 = zext i16 %1 to i32
+ %mul = mul nuw nsw i32 %conv2, %conv
+ %add = add nuw nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !14
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8COST,P9COST: LV: Checking a loop in 'dots16'
+; P8COST: Cost of 1 for VF 8: EXPRESSION vp<%8> = ir<%sum.08> + partial.reduce.add (mul nsw (ir<%1> sext to i32), (ir<%0> sext to i32))
+; P8COST: Cost for VF 8: 5 (Estimated cost per lane: 0.625)
+; P9COST: Cost of 2 for VF 8: EXPRESSION vp<%8> = ir<%sum.08> + partial.reduce.add (mul nsw (ir<%1> sext to i32), (ir<%0> sext to i32))
+; P9COST: Cost for VF 8: 8 (Estimated cost per lane: 1)
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %entry, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw [2 x i8], ptr %a, i64 %i.09
+ %0 = load i16, ptr %arrayidx, align 2, !tbaa !12
+ %conv = sext i16 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw [2 x i8], ptr %b, i64 %i.09
+ %1 = load i16, ptr %arrayidx1, align 2, !tbaa !12
+ %conv2 = sext i16 %1 to i32
+ %mul = mul nsw i32 %conv2, %conv
+ %add = add nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !15
+}
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+!llvm.errno.tbaa = !{!4}
+
+!0 = !{i32 8, !"PIC Level", i32 2}
+!1 = !{i32 7, !"PIE Level", i32 2}
+!2 = !{i32 7, !"uwtable", i32 2}
+!3 = !{!"clang version 23.0.0git"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C/C++ TBAA"}
+!8 = !{!6, !6, i64 0}
+!9 = distinct !{!9, !10}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = distinct !{!11, !10}
+!12 = !{!13, !13, i64 0}
+!13 = !{!"short", !6, i64 0}
+!14 = distinct !{!14, !10}
+!15 = distinct !{!15, !10}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
new file mode 100644
index 0000000000000..b3922c056ba9d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
@@ -0,0 +1,2003 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr8 -S -passes=loop-vectorize < %s | FileCheck --check-prefix=P8CHECK %s
+; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr9 -S -passes=loop-vectorize < %s | FileCheck --check-prefix=P9CHECK %s
+; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr10 -S -passes=loop-vectorize < %s | FileCheck --check-prefix=P10CHECK %s
+
+target datalayout = "e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8CHECK-LABEL: define dso_local signext i32 @dotu8(
+; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; P8CHECK-NEXT: [[ENTRY:.*]]:
+; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P8CHECK: [[ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P8CHECK: [[VECTOR_PH]]:
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 128
+; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P8CHECK: [[VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
+; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
+; P8CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64
+; P8CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 80
+; P8CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 96
+; P8CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 112
+; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA8:![0-9]+]]
+; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
+; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32
+; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48
+; P8CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 64
+; P8CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 80
+; P8CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 96
+; P8CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 112
+; P8CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD16]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP17:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <16 x i32> [[TMP16]], [[TMP17]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP18]])
+; P8CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD17]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <16 x i32> [[TMP19]], [[TMP20]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP21]])
+; P8CHECK-NEXT: [[TMP22:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <16 x i32> [[TMP22]], [[TMP23]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP24]])
+; P8CHECK-NEXT: [[TMP25:%.*]] = zext <16 x i8> [[WIDE_LOAD19]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP26:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <16 x i32> [[TMP25]], [[TMP26]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP27]])
+; P8CHECK-NEXT: [[TMP28:%.*]] = zext <16 x i8> [[WIDE_LOAD20]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <16 x i32> [[TMP28]], [[TMP29]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP30]])
+; P8CHECK-NEXT: [[TMP31:%.*]] = zext <16 x i8> [[WIDE_LOAD21]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP32:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <16 x i32> [[TMP31]], [[TMP32]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP33]])
+; P8CHECK-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD22]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <16 x i32> [[TMP34]], [[TMP35]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP36]])
+; P8CHECK-NEXT: [[TMP37:%.*]] = zext <16 x i8> [[WIDE_LOAD23]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP38:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <16 x i32> [[TMP37]], [[TMP38]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP39]])
+; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128
+; P8CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; P8CHECK: [[MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P8CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P8CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P8CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P8CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P8CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P8CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P8CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
+; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
+; P8CHECK: [[VEC_EPILOG_PH]]:
+; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 16
+; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]]
+; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]]
+; P8CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <16 x i8>, ptr [[TMP44]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP45:%.*]] = zext <16 x i8> [[WIDE_LOAD42]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP46:%.*]] = zext <16 x i8> [[WIDE_LOAD41]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <16 x i32> [[TMP45]], [[TMP46]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI40]], <16 x i32> [[TMP47]])
+; P8CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 16
+; P8CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P8CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P8CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P8CHECK: [[FOR_COND_CLEANUP]]:
+; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P8CHECK: [[FOR_BODY]]:
+; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32
+; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP51]] to i32
+; P8CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
+; P8CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]]
+; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+;
+; P9CHECK-LABEL: define dso_local signext i32 @dotu8(
+; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; P9CHECK-NEXT: [[ENTRY:.*]]:
+; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P9CHECK: [[ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P9CHECK: [[VECTOR_PH]]:
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 128
+; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P9CHECK: [[VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
+; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
+; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64
+; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 80
+; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 96
+; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 112
+; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA8:![0-9]+]]
+; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
+; P9CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32
+; P9CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48
+; P9CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 64
+; P9CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 80
+; P9CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 96
+; P9CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 112
+; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD16]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP17:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <16 x i32> [[TMP16]], [[TMP17]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP18]])
+; P9CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD17]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <16 x i32> [[TMP19]], [[TMP20]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP21]])
+; P9CHECK-NEXT: [[TMP22:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <16 x i32> [[TMP22]], [[TMP23]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP24]])
+; P9CHECK-NEXT: [[TMP25:%.*]] = zext <16 x i8> [[WIDE_LOAD19]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP26:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <16 x i32> [[TMP25]], [[TMP26]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP27]])
+; P9CHECK-NEXT: [[TMP28:%.*]] = zext <16 x i8> [[WIDE_LOAD20]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <16 x i32> [[TMP28]], [[TMP29]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP30]])
+; P9CHECK-NEXT: [[TMP31:%.*]] = zext <16 x i8> [[WIDE_LOAD21]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP32:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <16 x i32> [[TMP31]], [[TMP32]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP33]])
+; P9CHECK-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD22]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <16 x i32> [[TMP34]], [[TMP35]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP36]])
+; P9CHECK-NEXT: [[TMP37:%.*]] = zext <16 x i8> [[WIDE_LOAD23]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP38:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <16 x i32> [[TMP37]], [[TMP38]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP39]])
+; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128
+; P9CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; P9CHECK: [[MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P9CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P9CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P9CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P9CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P9CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P9CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P9CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
+; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
+; P9CHECK: [[VEC_EPILOG_PH]]:
+; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 16
+; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]]
+; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]]
+; P9CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <16 x i8>, ptr [[TMP44]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP45:%.*]] = zext <16 x i8> [[WIDE_LOAD42]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP46:%.*]] = zext <16 x i8> [[WIDE_LOAD41]] to <16 x i32>
+; P9CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <16 x i32> [[TMP45]], [[TMP46]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI40]], <16 x i32> [[TMP47]])
+; P9CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 16
+; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P9CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P9CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P9CHECK: [[FOR_COND_CLEANUP]]:
+; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P9CHECK: [[FOR_BODY]]:
+; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32
+; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP51]] to i32
+; P9CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
+; P9CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]]
+; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+;
+; P10CHECK-LABEL: define dso_local signext i32 @dotu8(
+; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; P10CHECK-NEXT: [[ENTRY:.*]]:
+; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P10CHECK: [[ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P10CHECK: [[VECTOR_PH]]:
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 128
+; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P10CHECK: [[VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
+; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
+; P10CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64
+; P10CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 80
+; P10CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 96
+; P10CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 112
+; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA8:![0-9]+]]
+; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
+; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32
+; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48
+; P10CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 64
+; P10CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 80
+; P10CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 96
+; P10CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 112
+; P10CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD16]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP17:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <16 x i32> [[TMP16]], [[TMP17]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP18]])
+; P10CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD17]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <16 x i32> [[TMP19]], [[TMP20]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP21]])
+; P10CHECK-NEXT: [[TMP22:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <16 x i32> [[TMP22]], [[TMP23]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP24]])
+; P10CHECK-NEXT: [[TMP25:%.*]] = zext <16 x i8> [[WIDE_LOAD19]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP26:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <16 x i32> [[TMP25]], [[TMP26]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP27]])
+; P10CHECK-NEXT: [[TMP28:%.*]] = zext <16 x i8> [[WIDE_LOAD20]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <16 x i32> [[TMP28]], [[TMP29]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP30]])
+; P10CHECK-NEXT: [[TMP31:%.*]] = zext <16 x i8> [[WIDE_LOAD21]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP32:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <16 x i32> [[TMP31]], [[TMP32]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP33]])
+; P10CHECK-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD22]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <16 x i32> [[TMP34]], [[TMP35]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP36]])
+; P10CHECK-NEXT: [[TMP37:%.*]] = zext <16 x i8> [[WIDE_LOAD23]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP38:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <16 x i32> [[TMP37]], [[TMP38]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP39]])
+; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128
+; P10CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; P10CHECK: [[MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P10CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P10CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P10CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P10CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P10CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P10CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P10CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
+; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
+; P10CHECK: [[VEC_EPILOG_PH]]:
+; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 16
+; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]]
+; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]]
+; P10CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <16 x i8>, ptr [[TMP44]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP45:%.*]] = zext <16 x i8> [[WIDE_LOAD42]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP46:%.*]] = zext <16 x i8> [[WIDE_LOAD41]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <16 x i32> [[TMP45]], [[TMP46]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI40]], <16 x i32> [[TMP47]])
+; P10CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 16
+; P10CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P10CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P10CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P10CHECK: [[FOR_COND_CLEANUP]]:
+; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P10CHECK: [[FOR_BODY]]:
+; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32
+; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP51]] to i32
+; P10CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
+; P10CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]]
+; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+;
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ %add.lcssa = phi i32 [ %add, %for.body ]
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.09
+ %0 = load i8, ptr %arrayidx, align 1, !tbaa !8
+ %conv = zext i8 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i64 %i.09
+ %1 = load i8, ptr %arrayidx1, align 1, !tbaa !8
+ %conv2 = zext i8 %1 to i32
+ %mul = mul nuw nsw i32 %conv2, %conv
+ %add = add nuw nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !9
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8CHECK-LABEL: define dso_local signext i32 @dots8(
+; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P8CHECK-NEXT: [[ENTRY:.*]]:
+; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P8CHECK: [[ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P8CHECK: [[VECTOR_PH]]:
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P8CHECK: [[VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
+; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
+; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD5]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD6]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD7]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
+; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32
+; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48
+; P8CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD8]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP13:%.*]] = sext <16 x i8> [[WIDE_LOAD9]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP14:%.*]] = sext <16 x i8> [[WIDE_LOAD10]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP15:%.*]] = sext <16 x i8> [[WIDE_LOAD11]] to <16 x i32>
+; P8CHECK-NEXT: [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP12]], [[TMP4]]
+; P8CHECK-NEXT: [[TMP17:%.*]] = mul nsw <16 x i32> [[TMP13]], [[TMP5]]
+; P8CHECK-NEXT: [[TMP18:%.*]] = mul nsw <16 x i32> [[TMP14]], [[TMP6]]
+; P8CHECK-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP15]], [[TMP7]]
+; P8CHECK-NEXT: [[TMP20]] = add <16 x i32> [[TMP16]], [[VEC_PHI]]
+; P8CHECK-NEXT: [[TMP21]] = add <16 x i32> [[TMP17]], [[VEC_PHI2]]
+; P8CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP18]], [[VEC_PHI3]]
+; P8CHECK-NEXT: [[TMP23]] = add <16 x i32> [[TMP19]], [[VEC_PHI4]]
+; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P8CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; P8CHECK: [[MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP21]], [[TMP20]]
+; P8CHECK-NEXT: [[BIN_RDX12:%.*]] = add <16 x i32> [[TMP22]], [[BIN_RDX]]
+; P8CHECK-NEXT: [[BIN_RDX13:%.*]] = add <16 x i32> [[TMP23]], [[BIN_RDX12]]
+; P8CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX13]])
+; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF17:![0-9]+]]
+; P8CHECK: [[VEC_EPILOG_PH]]:
+; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[N_MOD_VF14:%.*]] = urem i64 [[N]], 4
+; P8CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]]
+; P8CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX16]]
+; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP28:%.*]] = sext <4 x i8> [[WIDE_LOAD18]] to <4 x i32>
+; P8CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX16]]
+; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x i8>, ptr [[TMP29]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[TMP30:%.*]] = sext <4 x i8> [[WIDE_LOAD19]] to <4 x i32>
+; P8CHECK-NEXT: [[TMP31:%.*]] = mul nsw <4 x i32> [[TMP30]], [[TMP28]]
+; P8CHECK-NEXT: [[TMP32]] = add <4 x i32> [[TMP31]], [[VEC_PHI17]]
+; P8CHECK-NEXT: [[INDEX_NEXT20]] = add nuw i64 [[INDEX16]], 4
+; P8CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT20]], [[N_VEC15]]
+; P8CHECK-NEXT: br i1 [[TMP33]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[TMP34:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP32]])
+; P8CHECK-NEXT: [[CMP_N21:%.*]] = icmp eq i64 [[N]], [[N_VEC15]]
+; P8CHECK-NEXT: br i1 [[CMP_N21]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P8CHECK: [[FOR_COND_CLEANUP]]:
+; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P8CHECK: [[FOR_BODY]]:
+; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX22]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP35]] to i32
+; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P8CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP36]] to i32
+; P8CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
+; P8CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]]
+; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+;
+; P9CHECK-LABEL: define dso_local signext i32 @dots8(
+; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P9CHECK-NEXT: [[ENTRY:.*]]:
+; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P9CHECK: [[ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P9CHECK: [[VECTOR_PH]]:
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P9CHECK: [[VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP40:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP42:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP43:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP44:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP45:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP46:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP47:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24
+; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
+; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 40
+; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
+; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 56
+; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP8:%.*]] = sext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD9]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP10:%.*]] = sext <8 x i8> [[WIDE_LOAD10]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP11:%.*]] = sext <8 x i8> [[WIDE_LOAD11]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP12:%.*]] = sext <8 x i8> [[WIDE_LOAD12]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD13]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP14:%.*]] = sext <8 x i8> [[WIDE_LOAD14]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP15:%.*]] = sext <8 x i8> [[WIDE_LOAD15]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 8
+; P9CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 16
+; P9CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 24
+; P9CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 32
+; P9CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 40
+; P9CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 48
+; P9CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 56
+; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i8>, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i8>, ptr [[TMP17]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i8>, ptr [[TMP18]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i8>, ptr [[TMP19]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i8>, ptr [[TMP20]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i8>, ptr [[TMP21]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i8>, ptr [[TMP22]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i8>, ptr [[TMP23]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP24:%.*]] = sext <8 x i8> [[WIDE_LOAD16]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i8> [[WIDE_LOAD17]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i8> [[WIDE_LOAD18]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP27:%.*]] = sext <8 x i8> [[WIDE_LOAD19]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i8> [[WIDE_LOAD20]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i8> [[WIDE_LOAD21]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP30:%.*]] = sext <8 x i8> [[WIDE_LOAD22]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i8> [[WIDE_LOAD23]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP32:%.*]] = mul nsw <8 x i32> [[TMP24]], [[TMP8]]
+; P9CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP9]]
+; P9CHECK-NEXT: [[TMP34:%.*]] = mul nsw <8 x i32> [[TMP26]], [[TMP10]]
+; P9CHECK-NEXT: [[TMP35:%.*]] = mul nsw <8 x i32> [[TMP27]], [[TMP11]]
+; P9CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP12]]
+; P9CHECK-NEXT: [[TMP37:%.*]] = mul nsw <8 x i32> [[TMP29]], [[TMP13]]
+; P9CHECK-NEXT: [[TMP38:%.*]] = mul nsw <8 x i32> [[TMP30]], [[TMP14]]
+; P9CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP15]]
+; P9CHECK-NEXT: [[TMP40]] = add <8 x i32> [[TMP32]], [[VEC_PHI]]
+; P9CHECK-NEXT: [[TMP41]] = add <8 x i32> [[TMP33]], [[VEC_PHI2]]
+; P9CHECK-NEXT: [[TMP42]] = add <8 x i32> [[TMP34]], [[VEC_PHI3]]
+; P9CHECK-NEXT: [[TMP43]] = add <8 x i32> [[TMP35]], [[VEC_PHI4]]
+; P9CHECK-NEXT: [[TMP44]] = add <8 x i32> [[TMP36]], [[VEC_PHI5]]
+; P9CHECK-NEXT: [[TMP45]] = add <8 x i32> [[TMP37]], [[VEC_PHI6]]
+; P9CHECK-NEXT: [[TMP46]] = add <8 x i32> [[TMP38]], [[VEC_PHI7]]
+; P9CHECK-NEXT: [[TMP47]] = add <8 x i32> [[TMP39]], [[VEC_PHI8]]
+; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; P9CHECK: [[MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[TMP41]], [[TMP40]]
+; P9CHECK-NEXT: [[BIN_RDX24:%.*]] = add <8 x i32> [[TMP42]], [[BIN_RDX]]
+; P9CHECK-NEXT: [[BIN_RDX25:%.*]] = add <8 x i32> [[TMP43]], [[BIN_RDX24]]
+; P9CHECK-NEXT: [[BIN_RDX26:%.*]] = add <8 x i32> [[TMP44]], [[BIN_RDX25]]
+; P9CHECK-NEXT: [[BIN_RDX27:%.*]] = add <8 x i32> [[TMP45]], [[BIN_RDX26]]
+; P9CHECK-NEXT: [[BIN_RDX28:%.*]] = add <8 x i32> [[TMP46]], [[BIN_RDX27]]
+; P9CHECK-NEXT: [[BIN_RDX29:%.*]] = add <8 x i32> [[TMP47]], [[BIN_RDX28]]
+; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX29]])
+; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF17:![0-9]+]]
+; P9CHECK: [[VEC_EPILOG_PH]]:
+; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[N_MOD_VF30:%.*]] = urem i64 [[N]], 8
+; P9CHECK-NEXT: [[N_VEC31:%.*]] = sub i64 [[N]], [[N_MOD_VF30]]
+; P9CHECK-NEXT: [[TMP50:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX32:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT36:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI33:%.*]] = phi <8 x i32> [ [[TMP50]], %[[VEC_EPILOG_PH]] ], [ [[TMP56:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX32]]
+; P9CHECK-NEXT: [[WIDE_LOAD34:%.*]] = load <8 x i8>, ptr [[TMP51]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP52:%.*]] = sext <8 x i8> [[WIDE_LOAD34]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX32]]
+; P9CHECK-NEXT: [[WIDE_LOAD35:%.*]] = load <8 x i8>, ptr [[TMP53]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[TMP54:%.*]] = sext <8 x i8> [[WIDE_LOAD35]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP55:%.*]] = mul nsw <8 x i32> [[TMP54]], [[TMP52]]
+; P9CHECK-NEXT: [[TMP56]] = add <8 x i32> [[TMP55]], [[VEC_PHI33]]
+; P9CHECK-NEXT: [[INDEX_NEXT36]] = add nuw i64 [[INDEX32]], 8
+; P9CHECK-NEXT: [[TMP57:%.*]] = icmp eq i64 [[INDEX_NEXT36]], [[N_VEC31]]
+; P9CHECK-NEXT: br i1 [[TMP57]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[TMP58:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP56]])
+; P9CHECK-NEXT: [[CMP_N37:%.*]] = icmp eq i64 [[N]], [[N_VEC31]]
+; P9CHECK-NEXT: br i1 [[CMP_N37]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX38:%.*]] = phi i32 [ [[TMP58]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP49]], %[[MIDDLE_BLOCK]] ], [ [[TMP58]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P9CHECK: [[FOR_COND_CLEANUP]]:
+; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P9CHECK: [[FOR_BODY]]:
+; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX38]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP59:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP59]] to i32
+; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP60:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P9CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP60]] to i32
+; P9CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
+; P9CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]]
+; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+;
+; P10CHECK-LABEL: define dso_local signext i32 @dots8(
+; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P10CHECK-NEXT: [[ENTRY:.*]]:
+; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P10CHECK: [[ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P10CHECK: [[VECTOR_PH]]:
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P10CHECK: [[VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
+; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
+; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD5]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD6]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD7]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
+; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32
+; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48
+; P10CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD8]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP13:%.*]] = sext <16 x i8> [[WIDE_LOAD9]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP14:%.*]] = sext <16 x i8> [[WIDE_LOAD10]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP15:%.*]] = sext <16 x i8> [[WIDE_LOAD11]] to <16 x i32>
+; P10CHECK-NEXT: [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP12]], [[TMP4]]
+; P10CHECK-NEXT: [[TMP17:%.*]] = mul nsw <16 x i32> [[TMP13]], [[TMP5]]
+; P10CHECK-NEXT: [[TMP18:%.*]] = mul nsw <16 x i32> [[TMP14]], [[TMP6]]
+; P10CHECK-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP15]], [[TMP7]]
+; P10CHECK-NEXT: [[TMP20]] = add <16 x i32> [[TMP16]], [[VEC_PHI]]
+; P10CHECK-NEXT: [[TMP21]] = add <16 x i32> [[TMP17]], [[VEC_PHI2]]
+; P10CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP18]], [[VEC_PHI3]]
+; P10CHECK-NEXT: [[TMP23]] = add <16 x i32> [[TMP19]], [[VEC_PHI4]]
+; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P10CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; P10CHECK: [[MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP21]], [[TMP20]]
+; P10CHECK-NEXT: [[BIN_RDX12:%.*]] = add <16 x i32> [[TMP22]], [[BIN_RDX]]
+; P10CHECK-NEXT: [[BIN_RDX13:%.*]] = add <16 x i32> [[TMP23]], [[BIN_RDX12]]
+; P10CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX13]])
+; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF17:![0-9]+]]
+; P10CHECK: [[VEC_EPILOG_PH]]:
+; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[N_MOD_VF14:%.*]] = urem i64 [[N]], 4
+; P10CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]]
+; P10CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX16]]
+; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP28:%.*]] = sext <4 x i8> [[WIDE_LOAD18]] to <4 x i32>
+; P10CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX16]]
+; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x i8>, ptr [[TMP29]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[TMP30:%.*]] = sext <4 x i8> [[WIDE_LOAD19]] to <4 x i32>
+; P10CHECK-NEXT: [[TMP31:%.*]] = mul nsw <4 x i32> [[TMP30]], [[TMP28]]
+; P10CHECK-NEXT: [[TMP32]] = add <4 x i32> [[TMP31]], [[VEC_PHI17]]
+; P10CHECK-NEXT: [[INDEX_NEXT20]] = add nuw i64 [[INDEX16]], 4
+; P10CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT20]], [[N_VEC15]]
+; P10CHECK-NEXT: br i1 [[TMP33]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[TMP34:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP32]])
+; P10CHECK-NEXT: [[CMP_N21:%.*]] = icmp eq i64 [[N]], [[N_VEC15]]
+; P10CHECK-NEXT: br i1 [[CMP_N21]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P10CHECK: [[FOR_COND_CLEANUP]]:
+; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P10CHECK: [[FOR_BODY]]:
+; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX22]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP35]] to i32
+; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA8]]
+; P10CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP36]] to i32
+; P10CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
+; P10CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]]
+; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+;
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %entry, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.09
+ %0 = load i8, ptr %arrayidx, align 1, !tbaa !8
+ %conv = sext i8 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i64 %i.09
+ %1 = load i8, ptr %arrayidx1, align 1, !tbaa !8
+ %conv2 = sext i8 %1 to i32
+ %mul = mul nsw i32 %conv2, %conv
+ %add = add nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !11
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8CHECK-LABEL: define dso_local signext range(i32 0, -2147483648) i32 @dotu16(
+; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P8CHECK-NEXT: [[ENTRY:.*]]:
+; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P8CHECK: [[ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P8CHECK: [[VECTOR_PH]]:
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P8CHECK: [[VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8
+; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16
+; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24
+; P8CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32
+; P8CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40
+; P8CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48
+; P8CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56
+; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA20:![0-9]+]]
+; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8
+; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16
+; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24
+; P8CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32
+; P8CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40
+; P8CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48
+; P8CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56
+; P8CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i16> [[WIDE_LOAD16]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <8 x i32> [[TMP16]], [[TMP17]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]])
+; P8CHECK-NEXT: [[TMP19:%.*]] = zext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP20:%.*]] = zext <8 x i16> [[WIDE_LOAD9]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <8 x i32> [[TMP19]], [[TMP20]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]])
+; P8CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i16> [[WIDE_LOAD18]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP23:%.*]] = zext <8 x i16> [[WIDE_LOAD10]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <8 x i32> [[TMP22]], [[TMP23]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]])
+; P8CHECK-NEXT: [[TMP25:%.*]] = zext <8 x i16> [[WIDE_LOAD19]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP26:%.*]] = zext <8 x i16> [[WIDE_LOAD11]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]])
+; P8CHECK-NEXT: [[TMP28:%.*]] = zext <8 x i16> [[WIDE_LOAD20]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP29:%.*]] = zext <8 x i16> [[WIDE_LOAD12]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <8 x i32> [[TMP28]], [[TMP29]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]])
+; P8CHECK-NEXT: [[TMP31:%.*]] = zext <8 x i16> [[WIDE_LOAD21]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP32:%.*]] = zext <8 x i16> [[WIDE_LOAD13]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <8 x i32> [[TMP31]], [[TMP32]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]])
+; P8CHECK-NEXT: [[TMP34:%.*]] = zext <8 x i16> [[WIDE_LOAD22]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP35:%.*]] = zext <8 x i16> [[WIDE_LOAD14]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <8 x i32> [[TMP34]], [[TMP35]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]])
+; P8CHECK-NEXT: [[TMP37:%.*]] = zext <8 x i16> [[WIDE_LOAD23]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP38:%.*]] = zext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <8 x i32> [[TMP37]], [[TMP38]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]])
+; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P8CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; P8CHECK: [[MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P8CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P8CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P8CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P8CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P8CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P8CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P8CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF23:![0-9]+]]
+; P8CHECK: [[VEC_EPILOG_PH]]:
+; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
+; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
+; P8CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP45:%.*]] = zext <8 x i16> [[WIDE_LOAD42]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP46:%.*]] = zext <8 x i16> [[WIDE_LOAD41]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <8 x i32> [[TMP45]], [[TMP46]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]])
+; P8CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8
+; P8CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P8CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P8CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P8CHECK: [[FOR_COND_CLEANUP]]:
+; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P8CHECK: [[FOR_BODY]]:
+; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32
+; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[CONV2:%.*]] = zext i16 [[TMP51]] to i32
+; P8CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
+; P8CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]]
+; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+;
+; P9CHECK-LABEL: define dso_local signext range(i32 0, -2147483648) i32 @dotu16(
+; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P9CHECK-NEXT: [[ENTRY:.*]]:
+; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P9CHECK: [[ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P9CHECK: [[VECTOR_PH]]:
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P9CHECK: [[VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8
+; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16
+; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24
+; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32
+; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40
+; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48
+; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56
+; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA20:![0-9]+]]
+; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8
+; P9CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16
+; P9CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24
+; P9CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32
+; P9CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40
+; P9CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48
+; P9CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56
+; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i16> [[WIDE_LOAD16]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <8 x i32> [[TMP16]], [[TMP17]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]])
+; P9CHECK-NEXT: [[TMP19:%.*]] = zext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP20:%.*]] = zext <8 x i16> [[WIDE_LOAD9]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <8 x i32> [[TMP19]], [[TMP20]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]])
+; P9CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i16> [[WIDE_LOAD18]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP23:%.*]] = zext <8 x i16> [[WIDE_LOAD10]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <8 x i32> [[TMP22]], [[TMP23]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]])
+; P9CHECK-NEXT: [[TMP25:%.*]] = zext <8 x i16> [[WIDE_LOAD19]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP26:%.*]] = zext <8 x i16> [[WIDE_LOAD11]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]])
+; P9CHECK-NEXT: [[TMP28:%.*]] = zext <8 x i16> [[WIDE_LOAD20]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP29:%.*]] = zext <8 x i16> [[WIDE_LOAD12]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <8 x i32> [[TMP28]], [[TMP29]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]])
+; P9CHECK-NEXT: [[TMP31:%.*]] = zext <8 x i16> [[WIDE_LOAD21]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP32:%.*]] = zext <8 x i16> [[WIDE_LOAD13]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <8 x i32> [[TMP31]], [[TMP32]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]])
+; P9CHECK-NEXT: [[TMP34:%.*]] = zext <8 x i16> [[WIDE_LOAD22]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP35:%.*]] = zext <8 x i16> [[WIDE_LOAD14]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <8 x i32> [[TMP34]], [[TMP35]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]])
+; P9CHECK-NEXT: [[TMP37:%.*]] = zext <8 x i16> [[WIDE_LOAD23]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP38:%.*]] = zext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <8 x i32> [[TMP37]], [[TMP38]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]])
+; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P9CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; P9CHECK: [[MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P9CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P9CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P9CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P9CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P9CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P9CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P9CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF17]]
+; P9CHECK: [[VEC_EPILOG_PH]]:
+; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
+; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
+; P9CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP45:%.*]] = zext <8 x i16> [[WIDE_LOAD42]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP46:%.*]] = zext <8 x i16> [[WIDE_LOAD41]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <8 x i32> [[TMP45]], [[TMP46]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]])
+; P9CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8
+; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P9CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P9CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P9CHECK: [[FOR_COND_CLEANUP]]:
+; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P9CHECK: [[FOR_BODY]]:
+; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32
+; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[CONV2:%.*]] = zext i16 [[TMP51]] to i32
+; P9CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
+; P9CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]]
+; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+;
+; P10CHECK-LABEL: define dso_local signext range(i32 0, -2147483648) i32 @dotu16(
+; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P10CHECK-NEXT: [[ENTRY:.*]]:
+; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P10CHECK: [[ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P10CHECK: [[VECTOR_PH]]:
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P10CHECK: [[VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8
+; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16
+; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24
+; P10CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32
+; P10CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40
+; P10CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48
+; P10CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56
+; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA20:![0-9]+]]
+; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8
+; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16
+; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24
+; P10CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32
+; P10CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40
+; P10CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48
+; P10CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56
+; P10CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i16> [[WIDE_LOAD16]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <8 x i32> [[TMP16]], [[TMP17]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]])
+; P10CHECK-NEXT: [[TMP19:%.*]] = zext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP20:%.*]] = zext <8 x i16> [[WIDE_LOAD9]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <8 x i32> [[TMP19]], [[TMP20]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]])
+; P10CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i16> [[WIDE_LOAD18]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP23:%.*]] = zext <8 x i16> [[WIDE_LOAD10]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <8 x i32> [[TMP22]], [[TMP23]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]])
+; P10CHECK-NEXT: [[TMP25:%.*]] = zext <8 x i16> [[WIDE_LOAD19]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP26:%.*]] = zext <8 x i16> [[WIDE_LOAD11]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]])
+; P10CHECK-NEXT: [[TMP28:%.*]] = zext <8 x i16> [[WIDE_LOAD20]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP29:%.*]] = zext <8 x i16> [[WIDE_LOAD12]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <8 x i32> [[TMP28]], [[TMP29]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]])
+; P10CHECK-NEXT: [[TMP31:%.*]] = zext <8 x i16> [[WIDE_LOAD21]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP32:%.*]] = zext <8 x i16> [[WIDE_LOAD13]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <8 x i32> [[TMP31]], [[TMP32]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]])
+; P10CHECK-NEXT: [[TMP34:%.*]] = zext <8 x i16> [[WIDE_LOAD22]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP35:%.*]] = zext <8 x i16> [[WIDE_LOAD14]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <8 x i32> [[TMP34]], [[TMP35]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]])
+; P10CHECK-NEXT: [[TMP37:%.*]] = zext <8 x i16> [[WIDE_LOAD23]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP38:%.*]] = zext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <8 x i32> [[TMP37]], [[TMP38]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]])
+; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P10CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; P10CHECK: [[MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P10CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P10CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P10CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P10CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P10CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P10CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P10CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF23:![0-9]+]]
+; P10CHECK: [[VEC_EPILOG_PH]]:
+; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
+; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
+; P10CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP45:%.*]] = zext <8 x i16> [[WIDE_LOAD42]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP46:%.*]] = zext <8 x i16> [[WIDE_LOAD41]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <8 x i32> [[TMP45]], [[TMP46]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]])
+; P10CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8
+; P10CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P10CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P10CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P10CHECK: [[FOR_COND_CLEANUP]]:
+; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P10CHECK: [[FOR_BODY]]:
+; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32
+; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[CONV2:%.*]] = zext i16 [[TMP51]] to i32
+; P10CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
+; P10CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]]
+; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+;
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %entry, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw [2 x i8], ptr %a, i64 %i.09
+ %0 = load i16, ptr %arrayidx, align 2, !tbaa !12
+ %conv = zext i16 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw [2 x i8], ptr %b, i64 %i.09
+ %1 = load i16, ptr %arrayidx1, align 2, !tbaa !12
+ %conv2 = zext i16 %1 to i32
+ %mul = mul nuw nsw i32 %conv2, %conv
+ %add = add nuw nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !14
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable
+define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) local_unnamed_addr {
+; P8CHECK-LABEL: define dso_local signext i32 @dots16(
+; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P8CHECK-NEXT: [[ENTRY:.*]]:
+; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P8CHECK: [[ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P8CHECK: [[VECTOR_PH]]:
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P8CHECK: [[VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8
+; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16
+; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24
+; P8CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32
+; P8CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40
+; P8CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48
+; P8CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56
+; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]]
+; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8
+; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16
+; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24
+; P8CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32
+; P8CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40
+; P8CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48
+; P8CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56
+; P8CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i16> [[WIDE_LOAD16]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP17:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP18:%.*]] = mul nsw <8 x i32> [[TMP16]], [[TMP17]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]])
+; P8CHECK-NEXT: [[TMP19:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP20:%.*]] = sext <8 x i16> [[WIDE_LOAD9]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP21:%.*]] = mul nsw <8 x i32> [[TMP19]], [[TMP20]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]])
+; P8CHECK-NEXT: [[TMP22:%.*]] = sext <8 x i16> [[WIDE_LOAD18]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP23:%.*]] = sext <8 x i16> [[WIDE_LOAD10]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP24:%.*]] = mul nsw <8 x i32> [[TMP22]], [[TMP23]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]])
+; P8CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i16> [[WIDE_LOAD19]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i16> [[WIDE_LOAD11]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP27:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP26]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]])
+; P8CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i16> [[WIDE_LOAD20]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i16> [[WIDE_LOAD12]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP30:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP29]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]])
+; P8CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i16> [[WIDE_LOAD21]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP32:%.*]] = sext <8 x i16> [[WIDE_LOAD13]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP32]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]])
+; P8CHECK-NEXT: [[TMP34:%.*]] = sext <8 x i16> [[WIDE_LOAD22]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP35:%.*]] = sext <8 x i16> [[WIDE_LOAD14]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP34]], [[TMP35]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]])
+; P8CHECK-NEXT: [[TMP37:%.*]] = sext <8 x i16> [[WIDE_LOAD23]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP38:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP37]], [[TMP38]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]])
+; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P8CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; P8CHECK: [[MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P8CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P8CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P8CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P8CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P8CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P8CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P8CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF23]]
+; P8CHECK: [[VEC_EPILOG_PH]]:
+; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
+; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
+; P8CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[TMP45:%.*]] = sext <8 x i16> [[WIDE_LOAD42]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP46:%.*]] = sext <8 x i16> [[WIDE_LOAD41]] to <8 x i32>
+; P8CHECK-NEXT: [[TMP47:%.*]] = mul nsw <8 x i32> [[TMP45]], [[TMP46]]
+; P8CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]])
+; P8CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8
+; P8CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P8CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P8CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P8CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P8CHECK: [[FOR_COND_CLEANUP]]:
+; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P8CHECK: [[FOR_BODY]]:
+; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32
+; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]]
+; P8CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P8CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP51]] to i32
+; P8CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
+; P8CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]]
+; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+;
+; P9CHECK-LABEL: define dso_local signext i32 @dots16(
+; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P9CHECK-NEXT: [[ENTRY:.*]]:
+; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P9CHECK: [[ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P9CHECK: [[VECTOR_PH]]:
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P9CHECK: [[VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8
+; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16
+; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24
+; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32
+; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40
+; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48
+; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56
+; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]]
+; P9CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8
+; P9CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16
+; P9CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24
+; P9CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32
+; P9CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40
+; P9CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48
+; P9CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56
+; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i16> [[WIDE_LOAD16]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP17:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP18:%.*]] = mul nsw <8 x i32> [[TMP16]], [[TMP17]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]])
+; P9CHECK-NEXT: [[TMP19:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP20:%.*]] = sext <8 x i16> [[WIDE_LOAD9]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP21:%.*]] = mul nsw <8 x i32> [[TMP19]], [[TMP20]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]])
+; P9CHECK-NEXT: [[TMP22:%.*]] = sext <8 x i16> [[WIDE_LOAD18]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP23:%.*]] = sext <8 x i16> [[WIDE_LOAD10]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP24:%.*]] = mul nsw <8 x i32> [[TMP22]], [[TMP23]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]])
+; P9CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i16> [[WIDE_LOAD19]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i16> [[WIDE_LOAD11]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP27:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP26]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]])
+; P9CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i16> [[WIDE_LOAD20]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i16> [[WIDE_LOAD12]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP30:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP29]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]])
+; P9CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i16> [[WIDE_LOAD21]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP32:%.*]] = sext <8 x i16> [[WIDE_LOAD13]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP32]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]])
+; P9CHECK-NEXT: [[TMP34:%.*]] = sext <8 x i16> [[WIDE_LOAD22]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP35:%.*]] = sext <8 x i16> [[WIDE_LOAD14]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP34]], [[TMP35]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]])
+; P9CHECK-NEXT: [[TMP37:%.*]] = sext <8 x i16> [[WIDE_LOAD23]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP38:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP37]], [[TMP38]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]])
+; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P9CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; P9CHECK: [[MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P9CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P9CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P9CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P9CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P9CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P9CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P9CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF17]]
+; P9CHECK: [[VEC_EPILOG_PH]]:
+; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
+; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
+; P9CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[TMP45:%.*]] = sext <8 x i16> [[WIDE_LOAD42]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP46:%.*]] = sext <8 x i16> [[WIDE_LOAD41]] to <8 x i32>
+; P9CHECK-NEXT: [[TMP47:%.*]] = mul nsw <8 x i32> [[TMP45]], [[TMP46]]
+; P9CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]])
+; P9CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8
+; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P9CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P9CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P9CHECK: [[FOR_COND_CLEANUP]]:
+; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P9CHECK: [[FOR_BODY]]:
+; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32
+; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]]
+; P9CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P9CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP51]] to i32
+; P9CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
+; P9CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]]
+; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+;
+; P10CHECK-LABEL: define dso_local signext i32 @dots16(
+; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; P10CHECK-NEXT: [[ENTRY:.*]]:
+; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0
+; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; P10CHECK: [[ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
+; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; P10CHECK: [[VECTOR_PH]]:
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; P10CHECK: [[VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8
+; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16
+; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24
+; P10CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32
+; P10CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40
+; P10CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48
+; P10CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56
+; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]]
+; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8
+; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16
+; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24
+; P10CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32
+; P10CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40
+; P10CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48
+; P10CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56
+; P10CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i16> [[WIDE_LOAD16]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP17:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP18:%.*]] = mul nsw <8 x i32> [[TMP16]], [[TMP17]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]])
+; P10CHECK-NEXT: [[TMP19:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP20:%.*]] = sext <8 x i16> [[WIDE_LOAD9]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP21:%.*]] = mul nsw <8 x i32> [[TMP19]], [[TMP20]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]])
+; P10CHECK-NEXT: [[TMP22:%.*]] = sext <8 x i16> [[WIDE_LOAD18]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP23:%.*]] = sext <8 x i16> [[WIDE_LOAD10]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP24:%.*]] = mul nsw <8 x i32> [[TMP22]], [[TMP23]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]])
+; P10CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i16> [[WIDE_LOAD19]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i16> [[WIDE_LOAD11]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP27:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP26]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]])
+; P10CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i16> [[WIDE_LOAD20]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i16> [[WIDE_LOAD12]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP30:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP29]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]])
+; P10CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i16> [[WIDE_LOAD21]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP32:%.*]] = sext <8 x i16> [[WIDE_LOAD13]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP32]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]])
+; P10CHECK-NEXT: [[TMP34:%.*]] = sext <8 x i16> [[WIDE_LOAD22]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP35:%.*]] = sext <8 x i16> [[WIDE_LOAD14]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP34]], [[TMP35]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]])
+; P10CHECK-NEXT: [[TMP37:%.*]] = sext <8 x i16> [[WIDE_LOAD23]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP38:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP37]], [[TMP38]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]])
+; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; P10CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; P10CHECK: [[MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]]
+; P10CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]]
+; P10CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]]
+; P10CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]]
+; P10CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]]
+; P10CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]]
+; P10CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]]
+; P10CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]])
+; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF23]]
+; P10CHECK: [[VEC_EPILOG_PH]]:
+; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
+; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
+; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
+; P10CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[TMP45:%.*]] = sext <8 x i16> [[WIDE_LOAD42]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP46:%.*]] = sext <8 x i16> [[WIDE_LOAD41]] to <8 x i32>
+; P10CHECK-NEXT: [[TMP47:%.*]] = mul nsw <8 x i32> [[TMP45]], [[TMP46]]
+; P10CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]])
+; P10CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8
+; P10CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]]
+; P10CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; P10CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]])
+; P10CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]]
+; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; P10CHECK: [[FOR_COND_CLEANUP]]:
+; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+; P10CHECK: [[FOR_BODY]]:
+; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32
+; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]]
+; P10CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA20]]
+; P10CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP51]] to i32
+; P10CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
+; P10CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]]
+; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1
+; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+;
+entry:
+ %cmp7 = icmp sgt i64 %n, 0
+ br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ ret i32 %sum.0.lcssa
+
+for.body: ; preds = %entry, %for.body
+ %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+ %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw [2 x i8], ptr %a, i64 %i.09
+ %0 = load i16, ptr %arrayidx, align 2, !tbaa !12
+ %conv = sext i16 %0 to i32
+ %arrayidx1 = getelementptr inbounds nuw [2 x i8], ptr %b, i64 %i.09
+ %1 = load i16, ptr %arrayidx1, align 2, !tbaa !12
+ %conv2 = sext i16 %1 to i32
+ %mul = mul nsw i32 %conv2, %conv
+ %add = add nsw i32 %mul, %sum.08
+ %inc = add nuw nsw i64 %i.09, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !15
+}
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+!llvm.errno.tbaa = !{!4}
+
+!0 = !{i32 8, !"PIC Level", i32 2}
+!1 = !{i32 7, !"PIE Level", i32 2}
+!2 = !{i32 7, !"uwtable", i32 2}
+!3 = !{!"clang version 23.0.0git"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C/C++ TBAA"}
+!8 = !{!6, !6, i64 0}
+!9 = distinct !{!9, !10}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = distinct !{!11, !10}
+!12 = !{!13, !13, i64 0}
+!13 = !{!"short", !6, i64 0}
+!14 = distinct !{!14, !10}
+!15 = distinct !{!15, !10}
+;.
+; P8CHECK: [[META6:![0-9]+]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
+; P8CHECK: [[META7]] = !{!"Simple C/C++ TBAA"}
+; P8CHECK: [[CHAR_TBAA8]] = !{[[META6]], [[META6]], i64 0}
+; P8CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]], [[META12:![0-9]+]]}
+; P8CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
+; P8CHECK: [[META11]] = !{!"llvm.loop.isvectorized", i32 1}
+; P8CHECK: [[META12]] = !{!"llvm.loop.unroll.runtime.disable"}
+; P8CHECK: [[PROF13]] = !{!"branch_weights", i32 16, i32 112}
+; P8CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META10]], [[META11]], [[META12]]}
+; P8CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META10]], [[META12]], [[META11]]}
+; P8CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META10]], [[META11]], [[META12]]}
+; P8CHECK: [[PROF17]] = !{!"branch_weights", i32 4, i32 60}
+; P8CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META10]], [[META11]], [[META12]]}
+; P8CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META10]], [[META12]], [[META11]]}
+; P8CHECK: [[SHORT_TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; P8CHECK: [[META21]] = !{!"short", [[META6]], i64 0}
+; P8CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META10]], [[META11]], [[META12]]}
+; P8CHECK: [[PROF23]] = !{!"branch_weights", i32 8, i32 56}
+; P8CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META10]], [[META11]], [[META12]]}
+; P8CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META10]], [[META12]], [[META11]]}
+; P8CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META10]], [[META11]], [[META12]]}
+; P8CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META10]], [[META11]], [[META12]]}
+; P8CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META10]], [[META12]], [[META11]]}
+;.
+; P9CHECK: [[META6:![0-9]+]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
+; P9CHECK: [[META7]] = !{!"Simple C/C++ TBAA"}
+; P9CHECK: [[CHAR_TBAA8]] = !{[[META6]], [[META6]], i64 0}
+; P9CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]], [[META12:![0-9]+]]}
+; P9CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
+; P9CHECK: [[META11]] = !{!"llvm.loop.isvectorized", i32 1}
+; P9CHECK: [[META12]] = !{!"llvm.loop.unroll.runtime.disable"}
+; P9CHECK: [[PROF13]] = !{!"branch_weights", i32 16, i32 112}
+; P9CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META10]], [[META11]], [[META12]]}
+; P9CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META10]], [[META12]], [[META11]]}
+; P9CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META10]], [[META11]], [[META12]]}
+; P9CHECK: [[PROF17]] = !{!"branch_weights", i32 8, i32 56}
+; P9CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META10]], [[META11]], [[META12]]}
+; P9CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META10]], [[META12]], [[META11]]}
+; P9CHECK: [[SHORT_TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; P9CHECK: [[META21]] = !{!"short", [[META6]], i64 0}
+; P9CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META10]], [[META11]], [[META12]]}
+; P9CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META10]], [[META11]], [[META12]]}
+; P9CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META10]], [[META12]], [[META11]]}
+; P9CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META10]], [[META11]], [[META12]]}
+; P9CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META10]], [[META11]], [[META12]]}
+; P9CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META10]], [[META12]], [[META11]]}
+;.
+; P10CHECK: [[META6:![0-9]+]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
+; P10CHECK: [[META7]] = !{!"Simple C/C++ TBAA"}
+; P10CHECK: [[CHAR_TBAA8]] = !{[[META6]], [[META6]], i64 0}
+; P10CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]], [[META12:![0-9]+]]}
+; P10CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
+; P10CHECK: [[META11]] = !{!"llvm.loop.isvectorized", i32 1}
+; P10CHECK: [[META12]] = !{!"llvm.loop.unroll.runtime.disable"}
+; P10CHECK: [[PROF13]] = !{!"branch_weights", i32 16, i32 112}
+; P10CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META10]], [[META11]], [[META12]]}
+; P10CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META10]], [[META12]], [[META11]]}
+; P10CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META10]], [[META11]], [[META12]]}
+; P10CHECK: [[PROF17]] = !{!"branch_weights", i32 4, i32 60}
+; P10CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META10]], [[META11]], [[META12]]}
+; P10CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META10]], [[META12]], [[META11]]}
+; P10CHECK: [[SHORT_TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; P10CHECK: [[META21]] = !{!"short", [[META6]], i64 0}
+; P10CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META10]], [[META11]], [[META12]]}
+; P10CHECK: [[PROF23]] = !{!"branch_weights", i32 8, i32 56}
+; P10CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META10]], [[META11]], [[META12]]}
+; P10CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META10]], [[META12]], [[META11]]}
+; P10CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META10]], [[META11]], [[META12]]}
+; P10CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META10]], [[META11]], [[META12]]}
+; P10CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META10]], [[META12]], [[META11]]}
+;.
>From 439599ccfa6df2e6951e8f2214e3eb847d634443 Mon Sep 17 00:00:00 2001
From: Roland Froese <froese at ca.ibm.com>
Date: Fri, 7 Aug 2026 19:06:33 +0000
Subject: [PATCH 2/6] update test
---
.../LoopVectorize/PowerPC/partial-red.ll | 96 +++++++++----------
1 file changed, 48 insertions(+), 48 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
index b3922c056ba9d..faffd9582e7a9 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
@@ -20,7 +20,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128
; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P8CHECK: [[VECTOR_PH]]:
-; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 128
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 127
; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P8CHECK: [[VECTOR_BODY]]:
@@ -117,7 +117,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK: [[VEC_EPILOG_PH]]:
; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 16
+; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15
; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -141,7 +141,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P8CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -151,7 +151,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P8CHECK: [[FOR_BODY]]:
; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
; P8CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
; P8CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32
@@ -176,7 +176,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128
; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P9CHECK: [[VECTOR_PH]]:
-; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 128
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 127
; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P9CHECK: [[VECTOR_BODY]]:
@@ -273,7 +273,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK: [[VEC_EPILOG_PH]]:
; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 16
+; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15
; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -297,7 +297,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P9CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -307,7 +307,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P9CHECK: [[FOR_BODY]]:
; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
; P9CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
; P9CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32
@@ -332,7 +332,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128
; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P10CHECK: [[VECTOR_PH]]:
-; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 128
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 127
; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P10CHECK: [[VECTOR_BODY]]:
@@ -429,7 +429,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK: [[VEC_EPILOG_PH]]:
; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 16
+; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15
; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -453,7 +453,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P10CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -463,7 +463,7 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P10CHECK: [[FOR_BODY]]:
; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
; P10CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
; P10CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32
@@ -521,7 +521,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P8CHECK: [[VECTOR_PH]]:
-; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P8CHECK: [[VECTOR_BODY]]:
@@ -578,7 +578,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK: [[VEC_EPILOG_PH]]:
; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P8CHECK-NEXT: [[N_MOD_VF14:%.*]] = urem i64 [[N]], 4
+; P8CHECK-NEXT: [[N_MOD_VF14:%.*]] = and i64 [[N]], 3
; P8CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]]
; P8CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -602,7 +602,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: br i1 [[CMP_N21]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P8CHECK-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX21:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -612,7 +612,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P8CHECK: [[FOR_BODY]]:
; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX22]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX21]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
; P8CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
; P8CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP35]] to i32
@@ -637,7 +637,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P9CHECK: [[VECTOR_PH]]:
-; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P9CHECK: [[VECTOR_BODY]]:
@@ -734,7 +734,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK: [[VEC_EPILOG_PH]]:
; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P9CHECK-NEXT: [[N_MOD_VF30:%.*]] = urem i64 [[N]], 8
+; P9CHECK-NEXT: [[N_MOD_VF30:%.*]] = and i64 [[N]], 7
; P9CHECK-NEXT: [[N_VEC31:%.*]] = sub i64 [[N]], [[N_MOD_VF30]]
; P9CHECK-NEXT: [[TMP50:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -758,7 +758,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: br i1 [[CMP_N37]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P9CHECK-NEXT: [[BC_MERGE_RDX38:%.*]] = phi i32 [ [[TMP58]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX37:%.*]] = phi i32 [ [[TMP58]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP49]], %[[MIDDLE_BLOCK]] ], [ [[TMP58]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -768,7 +768,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P9CHECK: [[FOR_BODY]]:
; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX38]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX37]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
; P9CHECK-NEXT: [[TMP59:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
; P9CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP59]] to i32
@@ -793,7 +793,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P10CHECK: [[VECTOR_PH]]:
-; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P10CHECK: [[VECTOR_BODY]]:
@@ -850,7 +850,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK: [[VEC_EPILOG_PH]]:
; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P10CHECK-NEXT: [[N_MOD_VF14:%.*]] = urem i64 [[N]], 4
+; P10CHECK-NEXT: [[N_MOD_VF14:%.*]] = and i64 [[N]], 3
; P10CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]]
; P10CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -874,7 +874,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: br i1 [[CMP_N21]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P10CHECK-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX21:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -884,7 +884,7 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P10CHECK: [[FOR_BODY]]:
; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX22]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX21]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]]
; P10CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA8]]
; P10CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP35]] to i32
@@ -935,7 +935,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P8CHECK: [[VECTOR_PH]]:
-; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P8CHECK: [[VECTOR_BODY]]:
@@ -1032,7 +1032,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P8CHECK: [[VEC_EPILOG_PH]]:
; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -1056,7 +1056,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P8CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -1066,7 +1066,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P8CHECK: [[FOR_BODY]]:
; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
; P8CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
; P8CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32
@@ -1091,7 +1091,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P9CHECK: [[VECTOR_PH]]:
-; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P9CHECK: [[VECTOR_BODY]]:
@@ -1188,7 +1188,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P9CHECK: [[VEC_EPILOG_PH]]:
; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -1212,7 +1212,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P9CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -1222,7 +1222,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P9CHECK: [[FOR_BODY]]:
; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
; P9CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
; P9CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32
@@ -1247,7 +1247,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P10CHECK: [[VECTOR_PH]]:
-; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P10CHECK: [[VECTOR_BODY]]:
@@ -1344,7 +1344,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P10CHECK: [[VEC_EPILOG_PH]]:
; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -1368,7 +1368,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P10CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -1378,7 +1378,7 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P10CHECK: [[FOR_BODY]]:
; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
; P10CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
; P10CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32
@@ -1429,7 +1429,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P8CHECK: [[VECTOR_PH]]:
-; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P8CHECK: [[VECTOR_BODY]]:
@@ -1526,7 +1526,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P8CHECK: [[VEC_EPILOG_PH]]:
; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -1550,7 +1550,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P8CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P8CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P8CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -1560,7 +1560,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P8CHECK: [[FOR_BODY]]:
; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
; P8CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
; P8CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32
@@ -1585,7 +1585,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P9CHECK: [[VECTOR_PH]]:
-; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P9CHECK: [[VECTOR_BODY]]:
@@ -1682,7 +1682,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P9CHECK: [[VEC_EPILOG_PH]]:
; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -1706,7 +1706,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P9CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P9CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P9CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -1716,7 +1716,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P9CHECK: [[FOR_BODY]]:
; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
; P9CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
; P9CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32
@@ -1741,7 +1741,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64
; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; P10CHECK: [[VECTOR_PH]]:
-; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; P10CHECK: [[VECTOR_BODY]]:
@@ -1838,7 +1838,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P10CHECK: [[VEC_EPILOG_PH]]:
; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = urem i64 [[N]], 8
+; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -1862,7 +1862,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; P10CHECK-NEXT: [[BC_MERGE_RDX46:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; P10CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; P10CHECK-NEXT: br label %[[FOR_BODY:.*]]
; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -1872,7 +1872,7 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
; P10CHECK: [[FOR_BODY]]:
; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX46]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ]
; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]]
; P10CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA20]]
; P10CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32
>From c45f8493e01a2b700eddf2862c64b76d0cd0a9ab Mon Sep 17 00:00:00 2001
From: Roland Froese <froese at ca.ibm.com>
Date: Fri, 7 Aug 2026 19:21:08 +0000
Subject: [PATCH 3/6] fix formatting
---
llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index ea4539e08c672..94dedf55b08f2 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -1174,7 +1174,7 @@ InstructionCost PPCTTIImpl::getPartialReductionCost(
if (BinOp && BinOp.value() != Instruction::Mul)
return Invalid;
- EVT AccVT = TLI->getValueType(DL, AccumType, true);
+ EVT AccVT = TLI->getValueType(DL, AccumType, true);
if (AccVT != MVT::i32)
return Invalid;
if (InputTypeA != InputTypeB)
@@ -1193,7 +1193,7 @@ InstructionCost PPCTTIImpl::getPartialReductionCost(
if (AVT != MVT::v16i8 || OpBExtend != TTI::PR_ZeroExtend)
return Invalid;
} else if (AVT == MVT::v16i8 && OpAExtend == TTI::PR_SignExtend) {
- return Invalid;
+ return Invalid;
}
return vectorCostAdjustmentFactor(Instruction::Add, ATy, nullptr);
>From 7529d250569550d8321e8db51f369c2c43d0f12f Mon Sep 17 00:00:00 2001
From: Roland Froese <froese at ca.ibm.com>
Date: Mon, 24 Aug 2026 21:24:16 +0000
Subject: [PATCH 4/6] more formatting
---
llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 94dedf55b08f2..0d471ec1ef646 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -1165,8 +1165,7 @@ InstructionCost PPCTTIImpl::getPartialReductionCost(
unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
- TTI::TargetCostKind CostKind,
- std::optional<FastMathFlags> FMF) const {
+ TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
InstructionCost Invalid = InstructionCost::getInvalid();
if (Opcode != Instruction::Add)
>From 81c3d6e2211fb9b2a49d25bee277cabdd6316bc3 Mon Sep 17 00:00:00 2001
From: Roland Froese <froese at ca.ibm.com>
Date: Mon, 24 Aug 2026 21:39:17 +0000
Subject: [PATCH 5/6] and more formatting
---
llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 0d471ec1ef646..9bf05e6796935 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -1162,10 +1162,10 @@ PPCTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
}
InstructionCost PPCTTIImpl::getPartialReductionCost(
- unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
- ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
- TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
- TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
+ unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
+ ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
+ TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+ TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
InstructionCost Invalid = InstructionCost::getInvalid();
if (Opcode != Instruction::Add)
@@ -1180,7 +1180,7 @@ InstructionCost PPCTTIImpl::getPartialReductionCost(
return Invalid;
Type *ATy = VectorType::get(InputTypeA, VF);
- EVT AVT = TLI->getValueType(DL, ATy, true);
+ EVT AVT = TLI->getValueType(DL, ATy, true);
if (AVT != MVT::v16i8 && AVT != MVT::v8i16)
return Invalid;
>From a1f77e2eb5f6c3cc87d636f10ab7b042bd92b3ac Mon Sep 17 00:00:00 2001
From: Roland Froese <froese at ca.ibm.com>
Date: Tue, 25 Aug 2026 19:04:16 +0000
Subject: [PATCH 6/6] update test checks
---
.../LoopVectorize/PowerPC/partial-red.ll | 48 +++++++++----------
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
index faffd9582e7a9..f45410a5724c7 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll
@@ -119,11 +119,11 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15
; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]]
; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA8]]
; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]]
@@ -275,11 +275,11 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15
; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]]
; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA8]]
; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]]
@@ -431,11 +431,11 @@ define dso_local signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15
; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]]
; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA8]]
; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]]
@@ -580,11 +580,11 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[N_MOD_VF14:%.*]] = and i64 [[N]], 3
; P8CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]]
-; P8CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P8CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P8CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP37]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P8CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX16]]
; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1, !tbaa [[CHAR_TBAA8]]
; P8CHECK-NEXT: [[TMP28:%.*]] = sext <4 x i8> [[WIDE_LOAD18]] to <4 x i32>
@@ -736,11 +736,11 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[N_MOD_VF30:%.*]] = and i64 [[N]], 7
; P9CHECK-NEXT: [[N_VEC31:%.*]] = sub i64 [[N]], [[N_MOD_VF30]]
-; P9CHECK-NEXT: [[TMP50:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: [[TMP61:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P9CHECK-NEXT: [[INDEX32:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT36:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P9CHECK-NEXT: [[VEC_PHI33:%.*]] = phi <8 x i32> [ [[TMP50]], %[[VEC_EPILOG_PH]] ], [ [[TMP56:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI33:%.*]] = phi <8 x i32> [ [[TMP61]], %[[VEC_EPILOG_PH]] ], [ [[TMP56:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P9CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX32]]
; P9CHECK-NEXT: [[WIDE_LOAD34:%.*]] = load <8 x i8>, ptr [[TMP51]], align 1, !tbaa [[CHAR_TBAA8]]
; P9CHECK-NEXT: [[TMP52:%.*]] = sext <8 x i8> [[WIDE_LOAD34]] to <8 x i32>
@@ -852,11 +852,11 @@ define dso_local signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[N_MOD_VF14:%.*]] = and i64 [[N]], 3
; P10CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]]
-; P10CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P10CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P10CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP37]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P10CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX16]]
; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1, !tbaa [[CHAR_TBAA8]]
; P10CHECK-NEXT: [[TMP28:%.*]] = sext <4 x i8> [[WIDE_LOAD18]] to <4 x i32>
@@ -1034,11 +1034,11 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
@@ -1190,11 +1190,11 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
@@ -1346,11 +1346,11 @@ define dso_local signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef reado
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
@@ -1528,11 +1528,11 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P8CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P8CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
@@ -1684,11 +1684,11 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P9CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P9CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
@@ -1840,11 +1840,11 @@ define dso_local signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr
; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7
; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]]
-; P10CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; P10CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP42]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]]
; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA20]]
; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]]
More information about the llvm-commits
mailing list