[llvm] [WebAssembly] Lower v4i8 mul (PR #230052)
Sam Parker via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 8 03:25:31 PDT 2026
https://github.com/sparker-arm created https://github.com/llvm/llvm-project/pull/230052
Lower in the same manner as v8i8 and v16i8, which saves us one extlow instruction.
I've also moved one test out of codegen and into the vectorizer dir.
>From bcba7a66c8486cb88e9fcede30fa1d81a85abb5c Mon Sep 17 00:00:00 2001
From: Sam Parker <sam.parker at arm.com>
Date: Wed, 7 Oct 2026 11:36:22 +0100
Subject: [PATCH] [WebAssembly] Lower v4i8 mul
Lower in the same manner as v8i8 and v16i8, which saves us one extlow
instruction.
I've also moved one test out of codegen and into the vectorizer dir.
---
.../WebAssembly/WebAssemblyISelLowering.cpp | 30 +-
.../WebAssembly/int-mac-reduction-loops.ll | 10 +-
.../WebAssembly/interleaved-vectors.ll | 10 +-
.../CodeGen/WebAssembly/memory-interleave.ll | 5 +-
.../CodeGen/WebAssembly/quantized-dots.ll | 5 +-
.../CodeGen/WebAssembly/strided-int-mac.ll | 48 +-
.../WebAssembly/quantized-scalar-dots.ll | 1030 +++++++++++++++++
7 files changed, 1093 insertions(+), 45 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/WebAssembly/quantized-scalar-dots.ll
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index c11fe503490bc..a988093fd095f 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -3750,7 +3750,7 @@ static SDValue performSETCCCombine(SDNode *N,
static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG) {
EVT VT = N->getValueType(0);
- if (VT != MVT::v8i32 && VT != MVT::v16i32)
+ if (VT != MVT::v4i32 && VT != MVT::v8i32 && VT != MVT::v16i32)
return SDValue();
// Mul with extending inputs.
@@ -3790,6 +3790,8 @@ static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG) {
// | | %low_low = i32x4.ext_low_i16x8_ %low
// | | %low_high = i32x4.ext_high_i16x8_ %low
// | | %res = concat_vector(%low_low, %low_high)
+ // v4i8 | v4i32 | %low = i16x8.extmul_low_i8x16_ %a, %b
+ // | %res = i32x4.ext_low_i16x8_ %low
SDLoc DL(N);
unsigned NumElts = VT.getVectorNumElements();
@@ -3822,14 +3824,36 @@ static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG) {
GetExtendHigh(MVT::v4i32, MulHigh),
};
return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubVectors);
- } else {
- assert(NumElts == 8);
+ } else if (NumElts == 8) {
SDValue LowLHS = DAG.getNode(LHS->getOpcode(), DL, MVT::v8i16, ExtendInLHS);
SDValue LowRHS = DAG.getNode(RHS->getOpcode(), DL, MVT::v8i16, ExtendInRHS);
SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);
SDValue Lo = GetExtendLow(MVT::v4i32, MulLow);
SDValue Hi = GetExtendHigh(MVT::v4i32, MulLow);
return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
+ } else {
+ assert(NumElts == 4);
+ SDValue Poison = DAG.getPOISON(MVT::v4i8);
+ SDValue SubVectorsLHS[] = {
+ ExtendInLHS,
+ Poison,
+ Poison,
+ Poison,
+ };
+ SDValue SubVectorsRHS[] = {
+ ExtendInRHS,
+ Poison,
+ Poison,
+ Poison,
+ };
+ SDValue LowLHS =
+ GetExtendLow(MVT::v8i16, DAG.getNode(ISD::CONCAT_VECTORS, DL,
+ MVT::v16i8, SubVectorsLHS));
+ SDValue LowRHS =
+ GetExtendLow(MVT::v8i16, DAG.getNode(ISD::CONCAT_VECTORS, DL,
+ MVT::v16i8, SubVectorsRHS));
+ SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);
+ return GetExtendLow(MVT::v4i32, MulLow);
}
return SDValue();
}
diff --git a/llvm/test/CodeGen/WebAssembly/int-mac-reduction-loops.ll b/llvm/test/CodeGen/WebAssembly/int-mac-reduction-loops.ll
index 13fabb75e09f7..08fd561c23c4a 100644
--- a/llvm/test/CodeGen/WebAssembly/int-mac-reduction-loops.ll
+++ b/llvm/test/CodeGen/WebAssembly/int-mac-reduction-loops.ll
@@ -7,10 +7,9 @@ target triple = "wasm32"
define hidden i32 @i32_mac_s8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: i32_mac_s8:
; CHECK: v128.load32_zero 0:p2align=0
-; CHECK: i16x8.extend_low_i8x16_s
; CHECK: v128.load32_zero 0:p2align=0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extmul_low_i16x8_s
+; CHECK: i16x8.extmul_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
; CHECK: i32x4.add
; MAX-BANDWIDTH: v128.load
@@ -307,10 +306,9 @@ for.body: ; preds = %entry, %for.body
define hidden i32 @i32_mac_u8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: i32_mac_u8:
; CHECK: v128.load32_zero 0:p2align=0
-; CHECK: i16x8.extend_low_i8x16_u
; CHECK: v128.load32_zero 0:p2align=0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extmul_low_i16x8_u
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i32x4.extend_low_i16x8_u
; CHECK: i32x4.add
; MAX-BANDWIDTH: v128.load
diff --git a/llvm/test/CodeGen/WebAssembly/interleaved-vectors.ll b/llvm/test/CodeGen/WebAssembly/interleaved-vectors.ll
index d5934f8fde6cc..26c1f3e9470de 100644
--- a/llvm/test/CodeGen/WebAssembly/interleaved-vectors.ll
+++ b/llvm/test/CodeGen/WebAssembly/interleaved-vectors.ll
@@ -635,11 +635,10 @@ define hidden void @i32_i8_madd(ptr noalias nofree noundef writeonly captures(no
; CHECK-NEXT: local.get 9
; CHECK-NEXT: local.get 7
; CHECK-NEXT: v128.load32_zero 0:p2align=0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.get 6
; CHECK-NEXT: v128.load32_zero 0:p2align=0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 8
; CHECK-NEXT: v128.load32_zero 0:p2align=0
; CHECK-NEXT: i16x8.extend_low_i8x16_s
@@ -653,13 +652,12 @@ define hidden void @i32_i8_madd(ptr noalias nofree noundef writeonly captures(no
; CHECK-NEXT: i32.const 4
; CHECK-NEXT: i32.add
; CHECK-NEXT: v128.load32_zero 0:p2align=0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.get 6
; CHECK-NEXT: i32.const 4
; CHECK-NEXT: i32.add
; CHECK-NEXT: v128.load32_zero 0:p2align=0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 8
; CHECK-NEXT: i32.const 4
; CHECK-NEXT: i32.add
diff --git a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
index 63cee30414a7c..4ec619aaa109e 100644
--- a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
+++ b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
@@ -1291,10 +1291,9 @@ define hidden void @four_bytes_into_four_ints_same_op(ptr noalias nocapture noun
; CHECK: i32x4.sub
; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extmul_low_i16x8_u
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i32x4.extend_low_i16x8_u
; CHECK: v128.and
; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_u
diff --git a/llvm/test/CodeGen/WebAssembly/quantized-dots.ll b/llvm/test/CodeGen/WebAssembly/quantized-dots.ll
index 3fe30c95a2e64..992b89726d568 100644
--- a/llvm/test/CodeGen/WebAssembly/quantized-dots.ll
+++ b/llvm/test/CodeGen/WebAssembly/quantized-dots.ll
@@ -640,7 +640,6 @@ define hidden float @vec_dot_q6_K_q8_K(i32 noundef %arg, ptr noundef readonly %a
; CHECK-NEXT: local.get 2
; CHECK-NEXT: i32.add
; CHECK-NEXT: v128.load32_zero 0:p2align=0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.get 4
; CHECK-NEXT: local.get 5
; CHECK-NEXT: i32.const 4
@@ -649,8 +648,8 @@ define hidden float @vec_dot_q6_K_q8_K(i32 noundef %arg, ptr noundef readonly %a
; CHECK-NEXT: i32x4.extract_lane 0
; CHECK-NEXT: i32.add
; CHECK-NEXT: v128.load8_splat 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 0
; CHECK-NEXT: local.get 5
; CHECK-NEXT: i32.const 7
diff --git a/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll b/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll
index d96b89f9763d8..6bc51677d5924 100644
--- a/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll
+++ b/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll
@@ -88,16 +88,15 @@ define hidden { i32, i32, i32, i32 } @bb2053_inner_loop(ptr nocapture %base0, pt
; CHECK-NEXT: v128.load8_lane 0, 1
; CHECK-NEXT: v128.load8_lane 0, 2
; CHECK-NEXT: v128.load8_lane 0, 3
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 24
; CHECK-NEXT: local.get 3
; CHECK-NEXT: v128.load 0:p2align=0
; CHECK-NEXT: local.tee 13
; CHECK-NEXT: local.get 13
; CHECK-NEXT: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 25
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 14
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.get 7
@@ -116,32 +115,31 @@ define hidden { i32, i32, i32, i32 } @bb2053_inner_loop(ptr nocapture %base0, pt
; CHECK-NEXT: v128.load8_lane 0, 1
; CHECK-NEXT: v128.load8_lane 0, 2
; CHECK-NEXT: v128.load8_lane 0, 3
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 26
; CHECK-NEXT: local.get 13
; CHECK-NEXT: local.get 13
; CHECK-NEXT: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 27
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 14
; CHECK-NEXT: local.get 24
; CHECK-NEXT: local.get 13
; CHECK-NEXT: local.get 13
; CHECK-NEXT: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 28
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 16
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.get 26
; CHECK-NEXT: local.get 13
; CHECK-NEXT: local.get 13
; CHECK-NEXT: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 13
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 16
; CHECK-NEXT: local.get 25
@@ -165,9 +163,9 @@ define hidden { i32, i32, i32, i32 } @bb2053_inner_loop(ptr nocapture %base0, pt
; CHECK-NEXT: v128.load8_lane 0, 1
; CHECK-NEXT: v128.load8_lane 0, 2
; CHECK-NEXT: v128.load8_lane 0, 3
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 24
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 15
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.get 27
@@ -187,19 +185,21 @@ define hidden { i32, i32, i32, i32 } @bb2053_inner_loop(ptr nocapture %base0, pt
; CHECK-NEXT: v128.load8_lane 0, 1
; CHECK-NEXT: v128.load8_lane 0, 2
; CHECK-NEXT: v128.load8_lane 0, 3
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 25
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 15
; CHECK-NEXT: local.get 28
; CHECK-NEXT: local.get 24
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 17
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.get 13
; CHECK-NEXT: local.get 25
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 17
; CHECK-NEXT: local.get 3
@@ -1838,16 +1838,15 @@ define hidden { i32, i32, i32, i32 } @bb41_inner_loop(ptr nocapture %lhs, ptr no
; CHECK-NEXT: local.tee 15
; CHECK-NEXT: local.get 12
; CHECK-NEXT: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 16
; CHECK-NEXT: local.get 2
; CHECK-NEXT: v128.load64_zero 0:p2align=0
; CHECK-NEXT: local.tee 17
; CHECK-NEXT: local.get 12
; CHECK-NEXT: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 18
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 12
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 12
@@ -1855,9 +1854,9 @@ define hidden { i32, i32, i32, i32 } @bb41_inner_loop(ptr nocapture %lhs, ptr no
; CHECK-NEXT: local.get 17
; CHECK-NEXT: local.get 12
; CHECK-NEXT: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 17
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 13
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 13
@@ -1865,15 +1864,16 @@ define hidden { i32, i32, i32, i32 } @bb41_inner_loop(ptr nocapture %lhs, ptr no
; CHECK-NEXT: local.get 15
; CHECK-NEXT: local.get 12
; CHECK-NEXT: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 15
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 14
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 14
; CHECK-NEXT: local.get 17
; CHECK-NEXT: local.get 15
-; CHECK-NEXT: i32x4.extmul_low_i16x8_s
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s
+; CHECK-NEXT: i32x4.extend_low_i16x8_s
; CHECK-NEXT: local.get 11
; CHECK-NEXT: i32x4.add
; CHECK-NEXT: local.set 11
diff --git a/llvm/test/Transforms/LoopVectorize/WebAssembly/quantized-scalar-dots.ll b/llvm/test/Transforms/LoopVectorize/WebAssembly/quantized-scalar-dots.ll
new file mode 100644
index 0000000000000..c1a537d6adbb5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/WebAssembly/quantized-scalar-dots.ll
@@ -0,0 +1,1030 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=loop-vectorize %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=loop-vectorize --vectorizer-maximize-bandwidth %s -S -o - | FileCheck %s --check-prefix=MAX
+
+target datalayout="e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define hidden i32 @q4_0_loop(ptr noundef readonly captures(none) %arg, ptr noundef readonly captures(none) %arg1) local_unnamed_addr #0 {
+; CHECK-LABEL: define hidden i32 @q4_0_loop(
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA2:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[TMP1]], splat (i32 15)
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], splat (i32 -8)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i8> [[WIDE_LOAD1]] to <4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP6]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP8:%.*]] = lshr <4 x i32> [[TMP1]], splat (i32 4)
+; CHECK-NEXT: [[TMP9:%.*]] = add nsw <4 x i32> [[TMP8]], splat (i32 -8)
+; CHECK-NEXT: [[TMP10:%.*]] = or disjoint i32 [[INDEX]], 16
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP10]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i8> [[WIDE_LOAD2]] to <4 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <4 x i32> [[TMP9]], [[TMP12]]
+; CHECK-NEXT: [[TMP14]] = add <4 x i32> [[TMP7]], [[TMP13]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP14]])
+; CHECK-NEXT: br label %[[BB2:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: ret i32 [[TMP16]]
+;
+; MAX-LABEL: define hidden i32 @q4_0_loop(
+; MAX-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; MAX-NEXT: [[BB:.*:]]
+; MAX-NEXT: br label %[[VECTOR_PH:.*]]
+; MAX: [[VECTOR_PH]]:
+; MAX-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX: [[VECTOR_BODY]]:
+; MAX-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[ARG]], align 1, !tbaa [[CHAR_TBAA2:![0-9]+]]
+; MAX-NEXT: [[TMP0:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAX-NEXT: [[TMP1:%.*]] = and <16 x i32> [[TMP0]], splat (i32 15)
+; MAX-NEXT: [[TMP2:%.*]] = add nsw <16 x i32> [[TMP1]], splat (i32 -8)
+; MAX-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[ARG1]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; MAX-NEXT: [[TMP4:%.*]] = mul nsw <16 x i32> [[TMP2]], [[TMP3]]
+; MAX-NEXT: [[TMP5:%.*]] = add <16 x i32> [[TMP4]], zeroinitializer
+; MAX-NEXT: [[TMP6:%.*]] = lshr <16 x i32> [[TMP0]], splat (i32 4)
+; MAX-NEXT: [[TMP7:%.*]] = add nsw <16 x i32> [[TMP6]], splat (i32 -8)
+; MAX-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 16
+; MAX-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; MAX-NEXT: [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP9]]
+; MAX-NEXT: [[TMP11:%.*]] = add <16 x i32> [[TMP5]], [[TMP10]]
+; MAX-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; MAX: [[MIDDLE_BLOCK]]:
+; MAX-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP11]])
+; MAX-NEXT: br label %[[BB2:.*]]
+; MAX: [[BB2]]:
+; MAX-NEXT: ret i32 [[TMP12]]
+;
+bb:
+ br label %bb3
+
+bb2: ; preds = %bb3
+ ret i32 %i22
+
+bb3: ; preds = %bb3, %bb
+ %i = phi i32 [ 0, %bb ], [ %i22, %bb3 ]
+ %i4 = phi i32 [ 0, %bb ], [ %i23, %bb3 ]
+ %i5 = getelementptr inbounds i8, ptr %arg, i32 %i4
+ %i6 = load i8, ptr %i5, align 1, !tbaa !2
+ %i7 = zext i8 %i6 to i32
+ %i8 = and i32 %i7, 15
+ %i9 = add nsw i32 %i8, -8
+ %i10 = getelementptr inbounds i8, ptr %arg1, i32 %i4
+ %i11 = load i8, ptr %i10, align 1, !tbaa !2
+ %i12 = sext i8 %i11 to i32
+ %i13 = mul nsw i32 %i9, %i12
+ %i14 = add nsw i32 %i13, %i
+ %i15 = lshr i32 %i7, 4
+ %i16 = add nsw i32 %i15, -8
+ %i17 = or disjoint i32 %i4, 16
+ %i18 = getelementptr inbounds i8, ptr %arg1, i32 %i17
+ %i19 = load i8, ptr %i18, align 1, !tbaa !2
+ %i20 = sext i8 %i19 to i32
+ %i21 = mul nsw i32 %i16, %i20
+ %i22 = add nsw i32 %i14, %i21
+ %i23 = add nuw nsw i32 %i4, 1
+ %i24 = icmp eq i32 %i23, 16
+ br i1 %i24, label %bb2, label %bb3, !llvm.loop !5
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define hidden i32 @q2_dot_loop(ptr noundef readonly captures(none) %arg, ptr noundef readonly captures(none) %arg1, ptr noundef readonly captures(none) %arg2) local_unnamed_addr #0 {
+; CHECK-LABEL: define hidden i32 @q2_dot_loop(
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], ptr noundef readonly captures(none) [[ARG2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = lshr <4 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[TMP0]], splat (i32 32)
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 31)
+; CHECK-NEXT: [[TMP3:%.*]] = or disjoint <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 6)
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> poison, i8 [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP16]], i64 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP17]], i64 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP18]], i64 3
+; CHECK-NEXT: [[TMP23:%.*]] = zext <4 x i8> [[TMP22]] to <4 x i32>
+; CHECK-NEXT: [[TMP24:%.*]] = lshr <4 x i32> [[TMP23]], [[TMP6]]
+; CHECK-NEXT: [[TMP25:%.*]] = and <4 x i32> [[TMP24]], splat (i32 3)
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP5]]
+; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[TMP26]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP28:%.*]] = and i8 [[TMP27]], 15
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP28]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP29:%.*]] = zext nneg <4 x i8> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <4 x i32> [[TMP25]], [[TMP29]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP31]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP32:%.*]] = sext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP33:%.*]] = mul nsw <4 x i32> [[TMP30]], [[TMP32]]
+; CHECK-NEXT: [[TMP34]] = add <4 x i32> [[TMP33]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP36:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP34]])
+; CHECK-NEXT: br label %[[BB3:.*]]
+; CHECK: [[BB3]]:
+; CHECK-NEXT: ret i32 [[TMP36]]
+;
+; MAX-LABEL: define hidden i32 @q2_dot_loop(
+; MAX-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], ptr noundef readonly captures(none) [[ARG2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; MAX-NEXT: [[BB:.*:]]
+; MAX-NEXT: br label %[[VECTOR_PH:.*]]
+; MAX: [[VECTOR_PH]]:
+; MAX-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX: [[VECTOR_BODY]]:
+; MAX-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP82:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[VEC_IND:%.*]] = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[TMP0:%.*]] = lshr <16 x i32> [[VEC_IND]], splat (i32 2)
+; MAX-NEXT: [[TMP1:%.*]] = and <16 x i32> [[TMP0]], splat (i32 32)
+; MAX-NEXT: [[TMP2:%.*]] = and <16 x i32> [[VEC_IND]], splat (i32 31)
+; MAX-NEXT: [[TMP3:%.*]] = or disjoint <16 x i32> [[TMP1]], [[TMP2]]
+; MAX-NEXT: [[TMP4:%.*]] = lshr <16 x i32> [[VEC_IND]], splat (i32 4)
+; MAX-NEXT: [[TMP5:%.*]] = extractelement <16 x i32> [[TMP4]], i64 0
+; MAX-NEXT: [[TMP6:%.*]] = and <16 x i32> [[TMP4]], splat (i32 6)
+; MAX-NEXT: [[TMP7:%.*]] = extractelement <16 x i32> [[TMP3]], i64 0
+; MAX-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP7]]
+; MAX-NEXT: [[TMP9:%.*]] = extractelement <16 x i32> [[TMP3]], i64 1
+; MAX-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP9]]
+; MAX-NEXT: [[TMP11:%.*]] = extractelement <16 x i32> [[TMP3]], i64 2
+; MAX-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP11]]
+; MAX-NEXT: [[TMP13:%.*]] = extractelement <16 x i32> [[TMP3]], i64 3
+; MAX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP13]]
+; MAX-NEXT: [[TMP15:%.*]] = extractelement <16 x i32> [[TMP3]], i64 4
+; MAX-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP15]]
+; MAX-NEXT: [[TMP17:%.*]] = extractelement <16 x i32> [[TMP3]], i64 5
+; MAX-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP17]]
+; MAX-NEXT: [[TMP19:%.*]] = extractelement <16 x i32> [[TMP3]], i64 6
+; MAX-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP19]]
+; MAX-NEXT: [[TMP21:%.*]] = extractelement <16 x i32> [[TMP3]], i64 7
+; MAX-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP21]]
+; MAX-NEXT: [[TMP23:%.*]] = extractelement <16 x i32> [[TMP3]], i64 8
+; MAX-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP23]]
+; MAX-NEXT: [[TMP25:%.*]] = extractelement <16 x i32> [[TMP3]], i64 9
+; MAX-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP25]]
+; MAX-NEXT: [[TMP27:%.*]] = extractelement <16 x i32> [[TMP3]], i64 10
+; MAX-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP27]]
+; MAX-NEXT: [[TMP29:%.*]] = extractelement <16 x i32> [[TMP3]], i64 11
+; MAX-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP29]]
+; MAX-NEXT: [[TMP31:%.*]] = extractelement <16 x i32> [[TMP3]], i64 12
+; MAX-NEXT: [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP31]]
+; MAX-NEXT: [[TMP33:%.*]] = extractelement <16 x i32> [[TMP3]], i64 13
+; MAX-NEXT: [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP33]]
+; MAX-NEXT: [[TMP35:%.*]] = extractelement <16 x i32> [[TMP3]], i64 14
+; MAX-NEXT: [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP35]]
+; MAX-NEXT: [[TMP37:%.*]] = extractelement <16 x i32> [[TMP3]], i64 15
+; MAX-NEXT: [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP37]]
+; MAX-NEXT: [[TMP39:%.*]] = load i8, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP40:%.*]] = load i8, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP42:%.*]] = load i8, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP44:%.*]] = load i8, ptr [[TMP18]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP45:%.*]] = load i8, ptr [[TMP20]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP46:%.*]] = load i8, ptr [[TMP22]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP24]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP48:%.*]] = load i8, ptr [[TMP26]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP49:%.*]] = load i8, ptr [[TMP28]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP50:%.*]] = load i8, ptr [[TMP30]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP51:%.*]] = load i8, ptr [[TMP32]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP52:%.*]] = load i8, ptr [[TMP34]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP53:%.*]] = load i8, ptr [[TMP36]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP54:%.*]] = load i8, ptr [[TMP38]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP55:%.*]] = insertelement <16 x i8> poison, i8 [[TMP39]], i64 0
+; MAX-NEXT: [[TMP56:%.*]] = insertelement <16 x i8> [[TMP55]], i8 [[TMP40]], i64 1
+; MAX-NEXT: [[TMP57:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP41]], i64 2
+; MAX-NEXT: [[TMP58:%.*]] = insertelement <16 x i8> [[TMP57]], i8 [[TMP42]], i64 3
+; MAX-NEXT: [[TMP59:%.*]] = insertelement <16 x i8> [[TMP58]], i8 [[TMP43]], i64 4
+; MAX-NEXT: [[TMP60:%.*]] = insertelement <16 x i8> [[TMP59]], i8 [[TMP44]], i64 5
+; MAX-NEXT: [[TMP61:%.*]] = insertelement <16 x i8> [[TMP60]], i8 [[TMP45]], i64 6
+; MAX-NEXT: [[TMP62:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP46]], i64 7
+; MAX-NEXT: [[TMP63:%.*]] = insertelement <16 x i8> [[TMP62]], i8 [[TMP47]], i64 8
+; MAX-NEXT: [[TMP64:%.*]] = insertelement <16 x i8> [[TMP63]], i8 [[TMP48]], i64 9
+; MAX-NEXT: [[TMP65:%.*]] = insertelement <16 x i8> [[TMP64]], i8 [[TMP49]], i64 10
+; MAX-NEXT: [[TMP66:%.*]] = insertelement <16 x i8> [[TMP65]], i8 [[TMP50]], i64 11
+; MAX-NEXT: [[TMP67:%.*]] = insertelement <16 x i8> [[TMP66]], i8 [[TMP51]], i64 12
+; MAX-NEXT: [[TMP68:%.*]] = insertelement <16 x i8> [[TMP67]], i8 [[TMP52]], i64 13
+; MAX-NEXT: [[TMP69:%.*]] = insertelement <16 x i8> [[TMP68]], i8 [[TMP53]], i64 14
+; MAX-NEXT: [[TMP70:%.*]] = insertelement <16 x i8> [[TMP69]], i8 [[TMP54]], i64 15
+; MAX-NEXT: [[TMP71:%.*]] = zext <16 x i8> [[TMP70]] to <16 x i32>
+; MAX-NEXT: [[TMP72:%.*]] = lshr <16 x i32> [[TMP71]], [[TMP6]]
+; MAX-NEXT: [[TMP73:%.*]] = and <16 x i32> [[TMP72]], splat (i32 3)
+; MAX-NEXT: [[TMP74:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP5]]
+; MAX-NEXT: [[TMP75:%.*]] = load i8, ptr [[TMP74]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP76:%.*]] = and i8 [[TMP75]], 15
+; MAX-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[TMP76]], i64 0
+; MAX-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
+; MAX-NEXT: [[TMP77:%.*]] = zext nneg <16 x i8> [[BROADCAST_SPLAT]] to <16 x i32>
+; MAX-NEXT: [[TMP78:%.*]] = mul nuw nsw <16 x i32> [[TMP73]], [[TMP77]]
+; MAX-NEXT: [[TMP79:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[INDEX]]
+; MAX-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP79]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP80:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAX-NEXT: [[TMP81:%.*]] = mul nsw <16 x i32> [[TMP78]], [[TMP80]]
+; MAX-NEXT: [[TMP82]] = add <16 x i32> [[TMP81]], [[VEC_PHI]]
+; MAX-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; MAX-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i32> [[VEC_IND]], splat (i32 16)
+; MAX-NEXT: [[TMP83:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; MAX-NEXT: br i1 [[TMP83]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; MAX: [[MIDDLE_BLOCK]]:
+; MAX-NEXT: [[TMP84:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP82]])
+; MAX-NEXT: br label %[[BB3:.*]]
+; MAX: [[BB3]]:
+; MAX-NEXT: ret i32 [[TMP84]]
+;
+bb:
+ br label %bb4
+
+bb3: ; preds = %bb4
+ ret i32 %i26
+
+bb4: ; preds = %bb4, %bb
+ %i = phi i32 [ 0, %bb ], [ %i26, %bb4 ]
+ %i5 = phi i32 [ 0, %bb ], [ %i27, %bb4 ]
+ %i6 = lshr i32 %i5, 2
+ %i7 = and i32 %i6, 32
+ %i8 = and i32 %i5, 31
+ %i9 = or disjoint i32 %i7, %i8
+ %i10 = lshr i32 %i5, 4
+ %i11 = and i32 %i10, 6
+ %i12 = getelementptr inbounds i8, ptr %arg, i32 %i9
+ %i13 = load i8, ptr %i12, align 1, !tbaa !2
+ %i14 = zext i8 %i13 to i32
+ %i15 = lshr i32 %i14, %i11
+ %i16 = and i32 %i15, 3
+ %i17 = getelementptr inbounds i8, ptr %arg1, i32 %i10
+ %i18 = load i8, ptr %i17, align 1, !tbaa !2
+ %i19 = and i8 %i18, 15
+ %i20 = zext nneg i8 %i19 to i32
+ %i21 = mul nuw nsw i32 %i16, %i20
+ %i22 = getelementptr inbounds i8, ptr %arg2, i32 %i5
+ %i23 = load i8, ptr %i22, align 1, !tbaa !2
+ %i24 = sext i8 %i23 to i32
+ %i25 = mul nsw i32 %i21, %i24
+ %i26 = add nsw i32 %i25, %i
+ %i27 = add nuw nsw i32 %i5, 1
+ %i28 = icmp eq i32 %i27, 256
+ br i1 %i28, label %bb3, label %bb4, !llvm.loop !8
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define hidden i32 @min_loop(ptr noundef readonly captures(none) %arg, ptr noundef readonly captures(none) %arg1) local_unnamed_addr #0 {
+; CHECK-LABEL: define hidden i32 @min_loop(
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 4)
+; CHECK-NEXT: [[TMP2:%.*]] = zext nneg <4 x i8> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[ARG1]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA11:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext <4 x i16> [[WIDE_LOAD1]] to <4 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6]] = add <4 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[BB2:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: ret i32 [[TMP8]]
+;
+; MAX-LABEL: define hidden i32 @min_loop(
+; MAX-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; MAX-NEXT: [[BB:.*:]]
+; MAX-NEXT: br label %[[VECTOR_PH:.*]]
+; MAX: [[VECTOR_PH]]:
+; MAX-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX: [[VECTOR_BODY]]:
+; MAX-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[ARG]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP0:%.*]] = lshr <16 x i8> [[WIDE_LOAD]], splat (i8 4)
+; MAX-NEXT: [[TMP1:%.*]] = zext nneg <16 x i8> [[TMP0]] to <16 x i32>
+; MAX-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i16>, ptr [[ARG1]], align 2, !tbaa [[SHORT_TBAA10:![0-9]+]]
+; MAX-NEXT: [[TMP2:%.*]] = sext <16 x i16> [[WIDE_LOAD1]] to <16 x i32>
+; MAX-NEXT: [[TMP3:%.*]] = mul nsw <16 x i32> [[TMP1]], [[TMP2]]
+; MAX-NEXT: [[TMP4:%.*]] = add <16 x i32> [[TMP3]], zeroinitializer
+; MAX-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; MAX: [[MIDDLE_BLOCK]]:
+; MAX-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; MAX-NEXT: br label %[[BB2:.*]]
+; MAX: [[BB2]]:
+; MAX-NEXT: ret i32 [[TMP5]]
+;
+bb:
+ br label %bb3
+
+bb2: ; preds = %bb3
+ ret i32 %i13
+
+bb3: ; preds = %bb3, %bb
+ %i = phi i32 [ 0, %bb ], [ %i14, %bb3 ]
+ %i4 = phi i32 [ 0, %bb ], [ %i13, %bb3 ]
+ %i5 = getelementptr inbounds i8, ptr %arg, i32 %i
+ %i6 = load i8, ptr %i5, align 1, !tbaa !2
+ %i7 = lshr i8 %i6, 4
+ %i8 = zext nneg i8 %i7 to i32
+ %i9 = getelementptr inbounds i16, ptr %arg1, i32 %i
+ %i10 = load i16, ptr %i9, align 2, !tbaa !9
+ %i11 = sext i16 %i10 to i32
+ %i12 = mul nsw i32 %i8, %i11
+ %i13 = add nsw i32 %i12, %i4
+ %i14 = add nuw nsw i32 %i, 1
+ %i15 = icmp eq i32 %i14, 16
+ br i1 %i15, label %bb2, label %bb3, !llvm.loop !11
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define hidden i32 @q4k_loop(ptr noundef readonly captures(none) %arg, ptr noundef readonly captures(none) %arg1, i32 noundef %arg2, i32 noundef %arg3) local_unnamed_addr #0 {
+; CHECK-LABEL: define hidden i32 @q4k_loop(
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], i32 noundef [[ARG2:%.*]], i32 noundef [[ARG3:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[I:%.*]] = shl i32 [[ARG2]], 5
+; CHECK-NEXT: [[I4:%.*]] = lshr exact i32 [[I]], 1
+; CHECK-NEXT: [[I5:%.*]] = and i32 [[I4]], 2147483616
+; CHECK-NEXT: [[I6:%.*]] = shl i32 [[ARG2]], 2
+; CHECK-NEXT: [[I7:%.*]] = and i32 [[I6]], 4
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[I7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[ARG3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i32 [[INDEX]], [[I]]
+; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i32 [[INDEX]], [[I5]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i32> [[TMP4]], splat (i32 15)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP7:%.*]] = sext <4 x i8> [[WIDE_LOAD3]] to <4 x i32>
+; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP7]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT: [[TMP10]] = add <4 x i32> [[TMP9]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP10]])
+; CHECK-NEXT: br label %[[BB8:.*]]
+; CHECK: [[BB8]]:
+; CHECK-NEXT: ret i32 [[TMP12]]
+;
+; MAX-LABEL: define hidden i32 @q4k_loop(
+; MAX-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], i32 noundef [[ARG2:%.*]], i32 noundef [[ARG3:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; MAX-NEXT: [[BB:.*:]]
+; MAX-NEXT: [[I:%.*]] = shl i32 [[ARG2]], 5
+; MAX-NEXT: [[I4:%.*]] = lshr exact i32 [[I]], 1
+; MAX-NEXT: [[I5:%.*]] = and i32 [[I4]], 2147483616
+; MAX-NEXT: [[I6:%.*]] = shl i32 [[ARG2]], 2
+; MAX-NEXT: [[I7:%.*]] = and i32 [[I6]], 4
+; MAX-NEXT: br label %[[VECTOR_PH:.*]]
+; MAX: [[VECTOR_PH]]:
+; MAX-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[I7]], i64 0
+; MAX-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; MAX-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i32> poison, i32 [[ARG3]], i64 0
+; MAX-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT1]], <16 x i32> poison, <16 x i32> zeroinitializer
+; MAX-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX: [[VECTOR_BODY]]:
+; MAX-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[TMP0:%.*]] = or disjoint i32 [[INDEX]], [[I]]
+; MAX-NEXT: [[TMP1:%.*]] = or disjoint i32 [[INDEX]], [[I5]]
+; MAX-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP1]]
+; MAX-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAX-NEXT: [[TMP4:%.*]] = lshr <16 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
+; MAX-NEXT: [[TMP5:%.*]] = and <16 x i32> [[TMP4]], splat (i32 15)
+; MAX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP0]]
+; MAX-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD3]] to <16 x i32>
+; MAX-NEXT: [[TMP8:%.*]] = mul <16 x i32> [[TMP7]], [[BROADCAST_SPLAT2]]
+; MAX-NEXT: [[TMP9:%.*]] = mul <16 x i32> [[TMP8]], [[TMP5]]
+; MAX-NEXT: [[TMP10]] = add <16 x i32> [[TMP9]], [[VEC_PHI]]
+; MAX-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; MAX-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; MAX-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; MAX: [[MIDDLE_BLOCK]]:
+; MAX-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP10]])
+; MAX-NEXT: br label %[[BB8:.*]]
+; MAX: [[BB8]]:
+; MAX-NEXT: ret i32 [[TMP12]]
+;
+bb:
+ %i = shl i32 %arg2, 5
+ %i4 = lshr exact i32 %i, 1
+ %i5 = and i32 %i4, 2147483616
+ %i6 = shl i32 %arg2, 2
+ %i7 = and i32 %i6, 4
+ br label %bb9
+
+bb8: ; preds = %bb9
+ ret i32 %i24
+
+bb9: ; preds = %bb9, %bb
+ %i10 = phi i32 [ 0, %bb ], [ %i24, %bb9 ]
+ %i11 = phi i32 [ 0, %bb ], [ %i25, %bb9 ]
+ %i12 = or disjoint i32 %i11, %i
+ %i13 = or disjoint i32 %i11, %i5
+ %i14 = getelementptr inbounds i8, ptr %arg, i32 %i13
+ %i15 = load i8, ptr %i14, align 1, !tbaa !2
+ %i16 = zext i8 %i15 to i32
+ %i17 = lshr i32 %i16, %i7
+ %i18 = and i32 %i17, 15
+ %i19 = getelementptr inbounds i8, ptr %arg1, i32 %i12
+ %i20 = load i8, ptr %i19, align 1, !tbaa !2
+ %i21 = sext i8 %i20 to i32
+ %i22 = mul i32 %i21, %arg3
+ %i23 = mul i32 %i22, %i18
+ %i24 = add nsw i32 %i23, %i10
+ %i25 = add nuw nsw i32 %i11, 1
+ %i26 = icmp eq i32 %i25, 32
+ br i1 %i26, label %bb8, label %bb9, !llvm.loop !12
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define hidden i32 @q4_1_loop(ptr noundef readonly captures(none) %arg, ptr noundef readonly captures(none) %arg1) local_unnamed_addr #0 {
+; CHECK-LABEL: define hidden i32 @q4_1_loop(
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[TMP1]], splat (i32 15)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext <4 x i8> [[WIDE_LOAD1]] to <4 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP7:%.*]] = lshr <4 x i32> [[TMP1]], splat (i32 4)
+; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i32 [[INDEX]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP8]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP10:%.*]] = sext <4 x i8> [[WIDE_LOAD2]] to <4 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = mul nsw <4 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT: [[TMP12]] = add <4 x i32> [[TMP6]], [[TMP11]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP12]])
+; CHECK-NEXT: br label %[[BB2:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: ret i32 [[TMP14]]
+;
+; MAX-LABEL: define hidden i32 @q4_1_loop(
+; MAX-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; MAX-NEXT: [[BB:.*:]]
+; MAX-NEXT: br label %[[VECTOR_PH:.*]]
+; MAX: [[VECTOR_PH]]:
+; MAX-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX: [[VECTOR_BODY]]:
+; MAX-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[ARG]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP0:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAX-NEXT: [[TMP1:%.*]] = and <16 x i32> [[TMP0]], splat (i32 15)
+; MAX-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[ARG1]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP2:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; MAX-NEXT: [[TMP3:%.*]] = mul nsw <16 x i32> [[TMP1]], [[TMP2]]
+; MAX-NEXT: [[TMP4:%.*]] = add <16 x i32> [[TMP3]], zeroinitializer
+; MAX-NEXT: [[TMP5:%.*]] = lshr <16 x i32> [[TMP0]], splat (i32 4)
+; MAX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 16
+; MAX-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; MAX-NEXT: [[TMP8:%.*]] = mul nsw <16 x i32> [[TMP5]], [[TMP7]]
+; MAX-NEXT: [[TMP9:%.*]] = add <16 x i32> [[TMP4]], [[TMP8]]
+; MAX-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; MAX: [[MIDDLE_BLOCK]]:
+; MAX-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP9]])
+; MAX-NEXT: br label %[[BB2:.*]]
+; MAX: [[BB2]]:
+; MAX-NEXT: ret i32 [[TMP10]]
+;
+bb:
+ br label %bb3
+
+bb2: ; preds = %bb3
+ ret i32 %i20
+
+bb3: ; preds = %bb3, %bb
+ %i = phi i32 [ 0, %bb ], [ %i20, %bb3 ]
+ %i4 = phi i32 [ 0, %bb ], [ %i21, %bb3 ]
+ %i5 = getelementptr inbounds i8, ptr %arg, i32 %i4
+ %i6 = load i8, ptr %i5, align 1, !tbaa !2
+ %i7 = zext i8 %i6 to i32
+ %i8 = and i32 %i7, 15
+ %i9 = getelementptr inbounds i8, ptr %arg1, i32 %i4
+ %i10 = load i8, ptr %i9, align 1, !tbaa !2
+ %i11 = sext i8 %i10 to i32
+ %i12 = mul nsw i32 %i8, %i11
+ %i13 = add nsw i32 %i12, %i
+ %i14 = lshr i32 %i7, 4
+ %i15 = or disjoint i32 %i4, 16
+ %i16 = getelementptr inbounds i8, ptr %arg1, i32 %i15
+ %i17 = load i8, ptr %i16, align 1, !tbaa !2
+ %i18 = sext i8 %i17 to i32
+ %i19 = mul nsw i32 %i14, %i18
+ %i20 = add nsw i32 %i13, %i19
+ %i21 = add nuw nsw i32 %i4, 1
+ %i22 = icmp eq i32 %i21, 16
+ br i1 %i22, label %bb2, label %bb3, !llvm.loop !13
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define hidden i32 @q5k_loop(ptr noundef readonly captures(none) %arg, ptr noundef readonly captures(none) %arg1, ptr noundef readonly captures(none) %arg2, i32 noundef %arg3, i32 noundef %arg4) local_unnamed_addr #0 {
+; CHECK-LABEL: define hidden i32 @q5k_loop(
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], ptr noundef readonly captures(none) [[ARG2:%.*]], i32 noundef [[ARG3:%.*]], i32 noundef [[ARG4:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[I:%.*]] = shl i32 [[ARG3]], 5
+; CHECK-NEXT: [[I5:%.*]] = lshr i32 [[I]], 6
+; CHECK-NEXT: [[I6:%.*]] = and i32 [[ARG3]], 1
+; CHECK-NEXT: [[I7:%.*]] = shl nuw nsw i32 [[I5]], 5
+; CHECK-NEXT: [[I8:%.*]] = shl nuw nsw i32 [[I6]], 2
+; CHECK-NEXT: [[I9:%.*]] = shl nuw nsw i32 [[I5]], 1
+; CHECK-NEXT: [[I10:%.*]] = or disjoint i32 [[I9]], [[I6]]
+; CHECK-NEXT: [[I11:%.*]] = getelementptr i8, ptr [[ARG]], i32 [[I7]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[I8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[I10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[ARG4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i32 [[INDEX]], [[I]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[I11]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <4 x i32> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP4:%.*]] = and <4 x i32> [[TMP3]], splat (i32 15)
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[WIDE_LOAD5]] to <4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = lshr <4 x i32> [[TMP6]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw <4 x i32> [[TMP7]], splat (i32 4)
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i32> [[TMP8]], splat (i32 16)
+; CHECK-NEXT: [[TMP10:%.*]] = or disjoint <4 x i32> [[TMP9]], [[TMP4]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = mul <4 x i32> [[TMP12]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP14:%.*]] = mul <4 x i32> [[TMP13]], [[TMP10]]
+; CHECK-NEXT: [[TMP15]] = add <4 x i32> [[TMP14]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-NEXT: br label %[[BB12:.*]]
+; CHECK: [[BB12]]:
+; CHECK-NEXT: ret i32 [[TMP17]]
+;
+; MAX-LABEL: define hidden i32 @q5k_loop(
+; MAX-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], ptr noundef readonly captures(none) [[ARG2:%.*]], i32 noundef [[ARG3:%.*]], i32 noundef [[ARG4:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; MAX-NEXT: [[BB:.*:]]
+; MAX-NEXT: [[I:%.*]] = shl i32 [[ARG3]], 5
+; MAX-NEXT: [[I5:%.*]] = lshr i32 [[I]], 6
+; MAX-NEXT: [[I6:%.*]] = and i32 [[ARG3]], 1
+; MAX-NEXT: [[I7:%.*]] = shl nuw nsw i32 [[I5]], 5
+; MAX-NEXT: [[I8:%.*]] = shl nuw nsw i32 [[I6]], 2
+; MAX-NEXT: [[I9:%.*]] = shl nuw nsw i32 [[I5]], 1
+; MAX-NEXT: [[I10:%.*]] = or disjoint i32 [[I9]], [[I6]]
+; MAX-NEXT: [[I11:%.*]] = getelementptr i8, ptr [[ARG]], i32 [[I7]]
+; MAX-NEXT: br label %[[VECTOR_PH:.*]]
+; MAX: [[VECTOR_PH]]:
+; MAX-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[I8]], i64 0
+; MAX-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; MAX-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i32> poison, i32 [[I10]], i64 0
+; MAX-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT1]], <16 x i32> poison, <16 x i32> zeroinitializer
+; MAX-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <16 x i32> poison, i32 [[ARG4]], i64 0
+; MAX-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT3]], <16 x i32> poison, <16 x i32> zeroinitializer
+; MAX-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX: [[VECTOR_BODY]]:
+; MAX-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[TMP0:%.*]] = or disjoint i32 [[INDEX]], [[I]]
+; MAX-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[I11]], i32 [[INDEX]]
+; MAX-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; MAX-NEXT: [[TMP3:%.*]] = lshr <16 x i32> [[TMP2]], [[BROADCAST_SPLAT]]
+; MAX-NEXT: [[TMP4:%.*]] = and <16 x i32> [[TMP3]], splat (i32 15)
+; MAX-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[INDEX]]
+; MAX-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD5]] to <16 x i32>
+; MAX-NEXT: [[TMP7:%.*]] = lshr <16 x i32> [[TMP6]], [[BROADCAST_SPLAT2]]
+; MAX-NEXT: [[TMP8:%.*]] = shl nuw nsw <16 x i32> [[TMP7]], splat (i32 4)
+; MAX-NEXT: [[TMP9:%.*]] = and <16 x i32> [[TMP8]], splat (i32 16)
+; MAX-NEXT: [[TMP10:%.*]] = or disjoint <16 x i32> [[TMP9]], [[TMP4]]
+; MAX-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[TMP0]]
+; MAX-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD6]] to <16 x i32>
+; MAX-NEXT: [[TMP13:%.*]] = mul <16 x i32> [[TMP12]], [[BROADCAST_SPLAT4]]
+; MAX-NEXT: [[TMP14:%.*]] = mul <16 x i32> [[TMP13]], [[TMP10]]
+; MAX-NEXT: [[TMP15]] = add <16 x i32> [[TMP14]], [[VEC_PHI]]
+; MAX-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; MAX-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; MAX-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; MAX: [[MIDDLE_BLOCK]]:
+; MAX-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP15]])
+; MAX-NEXT: br label %[[BB12:.*]]
+; MAX: [[BB12]]:
+; MAX-NEXT: ret i32 [[TMP17]]
+;
+bb:
+ %i = shl i32 %arg3, 5
+ %i5 = lshr i32 %i, 6
+ %i6 = and i32 %arg3, 1
+ %i7 = shl nuw nsw i32 %i5, 5
+ %i8 = shl nuw nsw i32 %i6, 2
+ %i9 = shl nuw nsw i32 %i5, 1
+ %i10 = or disjoint i32 %i9, %i6
+ %i11 = getelementptr i8, ptr %arg, i32 %i7
+ br label %bb13
+
+bb12: ; preds = %bb13
+ ret i32 %i34
+
+bb13: ; preds = %bb13, %bb
+ %i14 = phi i32 [ 0, %bb ], [ %i34, %bb13 ]
+ %i15 = phi i32 [ 0, %bb ], [ %i35, %bb13 ]
+ %i16 = or disjoint i32 %i15, %i
+ %i17 = getelementptr i8, ptr %i11, i32 %i15
+ %i18 = load i8, ptr %i17, align 1, !tbaa !2
+ %i19 = zext i8 %i18 to i32
+ %i20 = lshr i32 %i19, %i8
+ %i21 = and i32 %i20, 15
+ %i22 = getelementptr inbounds i8, ptr %arg1, i32 %i15
+ %i23 = load i8, ptr %i22, align 1, !tbaa !2
+ %i24 = zext i8 %i23 to i32
+ %i25 = lshr i32 %i24, %i10
+ %i26 = shl nuw nsw i32 %i25, 4
+ %i27 = and i32 %i26, 16
+ %i28 = or disjoint i32 %i27, %i21
+ %i29 = getelementptr inbounds i8, ptr %arg2, i32 %i16
+ %i30 = load i8, ptr %i29, align 1, !tbaa !2
+ %i31 = sext i8 %i30 to i32
+ %i32 = mul i32 %i31, %arg4
+ %i33 = mul i32 %i32, %i28
+ %i34 = add nsw i32 %i33, %i14
+ %i35 = add nuw nsw i32 %i15, 1
+ %i36 = icmp eq i32 %i35, 32
+ br i1 %i36, label %bb12, label %bb13, !llvm.loop !14
+}
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define hidden i32 @q6k_loop(ptr noundef readonly captures(none) %arg, ptr noundef readonly captures(none) %arg1, ptr noundef readonly captures(none) %arg2, ptr noundef readonly captures(none) %arg3) local_unnamed_addr #0 {
+; CHECK-LABEL: define hidden i32 @q6k_loop(
+; CHECK-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], ptr noundef readonly captures(none) [[ARG2:%.*]], ptr noundef readonly captures(none) [[ARG3:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP59:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = lshr <4 x i32> [[VEC_IND]], splat (i32 7)
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw <4 x i32> [[TMP0]], splat (i32 6)
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 63)
+; CHECK-NEXT: [[TMP3:%.*]] = or disjoint <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw <4 x i32> [[TMP0]], splat (i32 5)
+; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 31)
+; CHECK-NEXT: [[TMP6:%.*]] = or disjoint <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = lshr <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i32> [[TMP7]], splat (i32 4)
+; CHECK-NEXT: [[TMP10:%.*]] = and <4 x i32> [[TMP7]], splat (i32 6)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP21:%.*]] = load i8, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP18]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i8> poison, i8 [[TMP19]], i64 0
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i8> [[TMP23]], i8 [[TMP20]], i64 1
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i8> [[TMP24]], i8 [[TMP21]], i64 2
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i8> [[TMP25]], i8 [[TMP22]], i64 3
+; CHECK-NEXT: [[TMP27:%.*]] = zext <4 x i8> [[TMP26]] to <4 x i32>
+; CHECK-NEXT: [[TMP28:%.*]] = lshr <4 x i32> [[TMP27]], [[TMP9]]
+; CHECK-NEXT: [[TMP29:%.*]] = and <4 x i32> [[TMP28]], splat (i32 15)
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP30]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP6]], i64 1
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP32]]
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i32> [[TMP6]], i64 2
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP34]]
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <4 x i32> [[TMP6]], i64 3
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = load i8, ptr [[TMP31]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP39:%.*]] = load i8, ptr [[TMP33]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP40:%.*]] = load i8, ptr [[TMP35]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP37]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i8> poison, i8 [[TMP38]], i64 0
+; CHECK-NEXT: [[TMP43:%.*]] = insertelement <4 x i8> [[TMP42]], i8 [[TMP39]], i64 1
+; CHECK-NEXT: [[TMP44:%.*]] = insertelement <4 x i8> [[TMP43]], i8 [[TMP40]], i64 2
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x i8> [[TMP44]], i8 [[TMP41]], i64 3
+; CHECK-NEXT: [[TMP46:%.*]] = zext <4 x i8> [[TMP45]] to <4 x i32>
+; CHECK-NEXT: [[TMP47:%.*]] = lshr <4 x i32> [[TMP46]], [[TMP10]]
+; CHECK-NEXT: [[TMP48:%.*]] = shl nuw nsw <4 x i32> [[TMP47]], splat (i32 4)
+; CHECK-NEXT: [[TMP49:%.*]] = and <4 x i32> [[TMP48]], splat (i32 48)
+; CHECK-NEXT: [[TMP50:%.*]] = or disjoint <4 x i32> [[TMP29]], splat (i32 -32)
+; CHECK-NEXT: [[TMP51:%.*]] = add nsw <4 x i32> [[TMP50]], [[TMP49]]
+; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[TMP8]]
+; CHECK-NEXT: [[TMP53:%.*]] = load i8, ptr [[TMP52]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP53]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP54:%.*]] = sext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds i8, ptr [[ARG3]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP55]], align 1, !tbaa [[CHAR_TBAA2]]
+; CHECK-NEXT: [[TMP56:%.*]] = sext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP57:%.*]] = mul nsw <4 x i32> [[TMP56]], [[TMP54]]
+; CHECK-NEXT: [[TMP58:%.*]] = mul nsw <4 x i32> [[TMP57]], [[TMP51]]
+; CHECK-NEXT: [[TMP59]] = add <4 x i32> [[TMP58]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP60:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: br i1 [[TMP60]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP61:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP59]])
+; CHECK-NEXT: br label %[[BB4:.*]]
+; CHECK: [[BB4]]:
+; CHECK-NEXT: ret i32 [[TMP61]]
+;
+; MAX-LABEL: define hidden i32 @q6k_loop(
+; MAX-SAME: ptr noundef readonly captures(none) [[ARG:%.*]], ptr noundef readonly captures(none) [[ARG1:%.*]], ptr noundef readonly captures(none) [[ARG2:%.*]], ptr noundef readonly captures(none) [[ARG3:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; MAX-NEXT: [[BB:.*:]]
+; MAX-NEXT: br label %[[VECTOR_PH:.*]]
+; MAX: [[VECTOR_PH]]:
+; MAX-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX: [[VECTOR_BODY]]:
+; MAX-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP91:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MAX-NEXT: [[TMP0:%.*]] = lshr <8 x i32> [[VEC_IND]], splat (i32 7)
+; MAX-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[TMP0]], splat (i32 6)
+; MAX-NEXT: [[TMP2:%.*]] = and <8 x i32> [[VEC_IND]], splat (i32 63)
+; MAX-NEXT: [[TMP3:%.*]] = or disjoint <8 x i32> [[TMP1]], [[TMP2]]
+; MAX-NEXT: [[TMP4:%.*]] = shl nuw nsw <8 x i32> [[TMP0]], splat (i32 5)
+; MAX-NEXT: [[TMP5:%.*]] = and <8 x i32> [[VEC_IND]], splat (i32 31)
+; MAX-NEXT: [[TMP6:%.*]] = or disjoint <8 x i32> [[TMP4]], [[TMP5]]
+; MAX-NEXT: [[TMP7:%.*]] = lshr <8 x i32> [[VEC_IND]], splat (i32 4)
+; MAX-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP7]], i64 0
+; MAX-NEXT: [[TMP9:%.*]] = and <8 x i32> [[TMP7]], splat (i32 4)
+; MAX-NEXT: [[TMP10:%.*]] = and <8 x i32> [[TMP7]], splat (i32 6)
+; MAX-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; MAX-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP11]]
+; MAX-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
+; MAX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP13]]
+; MAX-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
+; MAX-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP15]]
+; MAX-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
+; MAX-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP17]]
+; MAX-NEXT: [[TMP19:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
+; MAX-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP19]]
+; MAX-NEXT: [[TMP21:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
+; MAX-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP21]]
+; MAX-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
+; MAX-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP23]]
+; MAX-NEXT: [[TMP25:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
+; MAX-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[TMP25]]
+; MAX-NEXT: [[TMP27:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP28:%.*]] = load i8, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP29:%.*]] = load i8, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP30:%.*]] = load i8, ptr [[TMP18]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP31:%.*]] = load i8, ptr [[TMP20]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP22]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP24]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP26]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP35:%.*]] = insertelement <8 x i8> poison, i8 [[TMP27]], i64 0
+; MAX-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP35]], i8 [[TMP28]], i64 1
+; MAX-NEXT: [[TMP37:%.*]] = insertelement <8 x i8> [[TMP36]], i8 [[TMP29]], i64 2
+; MAX-NEXT: [[TMP38:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP30]], i64 3
+; MAX-NEXT: [[TMP39:%.*]] = insertelement <8 x i8> [[TMP38]], i8 [[TMP31]], i64 4
+; MAX-NEXT: [[TMP40:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[TMP32]], i64 5
+; MAX-NEXT: [[TMP41:%.*]] = insertelement <8 x i8> [[TMP40]], i8 [[TMP33]], i64 6
+; MAX-NEXT: [[TMP42:%.*]] = insertelement <8 x i8> [[TMP41]], i8 [[TMP34]], i64 7
+; MAX-NEXT: [[TMP43:%.*]] = zext <8 x i8> [[TMP42]] to <8 x i32>
+; MAX-NEXT: [[TMP44:%.*]] = lshr <8 x i32> [[TMP43]], [[TMP9]]
+; MAX-NEXT: [[TMP45:%.*]] = and <8 x i32> [[TMP44]], splat (i32 15)
+; MAX-NEXT: [[TMP46:%.*]] = extractelement <8 x i32> [[TMP6]], i64 0
+; MAX-NEXT: [[TMP47:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP46]]
+; MAX-NEXT: [[TMP48:%.*]] = extractelement <8 x i32> [[TMP6]], i64 1
+; MAX-NEXT: [[TMP49:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP48]]
+; MAX-NEXT: [[TMP50:%.*]] = extractelement <8 x i32> [[TMP6]], i64 2
+; MAX-NEXT: [[TMP51:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP50]]
+; MAX-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP6]], i64 3
+; MAX-NEXT: [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP52]]
+; MAX-NEXT: [[TMP54:%.*]] = extractelement <8 x i32> [[TMP6]], i64 4
+; MAX-NEXT: [[TMP55:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP54]]
+; MAX-NEXT: [[TMP56:%.*]] = extractelement <8 x i32> [[TMP6]], i64 5
+; MAX-NEXT: [[TMP57:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP56]]
+; MAX-NEXT: [[TMP58:%.*]] = extractelement <8 x i32> [[TMP6]], i64 6
+; MAX-NEXT: [[TMP59:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP58]]
+; MAX-NEXT: [[TMP60:%.*]] = extractelement <8 x i32> [[TMP6]], i64 7
+; MAX-NEXT: [[TMP61:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i32 [[TMP60]]
+; MAX-NEXT: [[TMP62:%.*]] = load i8, ptr [[TMP47]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP63:%.*]] = load i8, ptr [[TMP49]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP64:%.*]] = load i8, ptr [[TMP51]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP65:%.*]] = load i8, ptr [[TMP53]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP66:%.*]] = load i8, ptr [[TMP55]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP67:%.*]] = load i8, ptr [[TMP57]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP68:%.*]] = load i8, ptr [[TMP59]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP69:%.*]] = load i8, ptr [[TMP61]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP70:%.*]] = insertelement <8 x i8> poison, i8 [[TMP62]], i64 0
+; MAX-NEXT: [[TMP71:%.*]] = insertelement <8 x i8> [[TMP70]], i8 [[TMP63]], i64 1
+; MAX-NEXT: [[TMP72:%.*]] = insertelement <8 x i8> [[TMP71]], i8 [[TMP64]], i64 2
+; MAX-NEXT: [[TMP73:%.*]] = insertelement <8 x i8> [[TMP72]], i8 [[TMP65]], i64 3
+; MAX-NEXT: [[TMP74:%.*]] = insertelement <8 x i8> [[TMP73]], i8 [[TMP66]], i64 4
+; MAX-NEXT: [[TMP75:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP67]], i64 5
+; MAX-NEXT: [[TMP76:%.*]] = insertelement <8 x i8> [[TMP75]], i8 [[TMP68]], i64 6
+; MAX-NEXT: [[TMP77:%.*]] = insertelement <8 x i8> [[TMP76]], i8 [[TMP69]], i64 7
+; MAX-NEXT: [[TMP78:%.*]] = zext <8 x i8> [[TMP77]] to <8 x i32>
+; MAX-NEXT: [[TMP79:%.*]] = lshr <8 x i32> [[TMP78]], [[TMP10]]
+; MAX-NEXT: [[TMP80:%.*]] = shl nuw nsw <8 x i32> [[TMP79]], splat (i32 4)
+; MAX-NEXT: [[TMP81:%.*]] = and <8 x i32> [[TMP80]], splat (i32 48)
+; MAX-NEXT: [[TMP82:%.*]] = or disjoint <8 x i32> [[TMP45]], splat (i32 -32)
+; MAX-NEXT: [[TMP83:%.*]] = add nsw <8 x i32> [[TMP82]], [[TMP81]]
+; MAX-NEXT: [[TMP84:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[TMP8]]
+; MAX-NEXT: [[TMP85:%.*]] = load i8, ptr [[TMP84]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP85]], i64 0
+; MAX-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; MAX-NEXT: [[TMP86:%.*]] = sext <8 x i8> [[BROADCAST_SPLAT]] to <8 x i32>
+; MAX-NEXT: [[TMP87:%.*]] = getelementptr inbounds i8, ptr [[ARG3]], i32 [[INDEX]]
+; MAX-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP87]], align 1, !tbaa [[CHAR_TBAA2]]
+; MAX-NEXT: [[TMP88:%.*]] = sext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; MAX-NEXT: [[TMP89:%.*]] = mul nsw <8 x i32> [[TMP88]], [[TMP86]]
+; MAX-NEXT: [[TMP90:%.*]] = mul nsw <8 x i32> [[TMP89]], [[TMP83]]
+; MAX-NEXT: [[TMP91]] = add <8 x i32> [[TMP90]], [[VEC_PHI]]
+; MAX-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; MAX-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
+; MAX-NEXT: [[TMP92:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; MAX-NEXT: br i1 [[TMP92]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; MAX: [[MIDDLE_BLOCK]]:
+; MAX-NEXT: [[TMP93:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP91]])
+; MAX-NEXT: br label %[[BB4:.*]]
+; MAX: [[BB4]]:
+; MAX-NEXT: ret i32 [[TMP93]]
+;
+bb:
+ br label %bb5
+
+bb4: ; preds = %bb5
+ ret i32 %i38
+
+bb5: ; preds = %bb5, %bb
+ %i = phi i32 [ 0, %bb ], [ %i38, %bb5 ]
+ %i6 = phi i32 [ 0, %bb ], [ %i39, %bb5 ]
+ %i7 = lshr i32 %i6, 7
+ %i8 = shl nuw nsw i32 %i7, 6
+ %i9 = and i32 %i6, 63
+ %i10 = or disjoint i32 %i8, %i9
+ %i11 = shl nuw nsw i32 %i7, 5
+ %i12 = and i32 %i6, 31
+ %i13 = or disjoint i32 %i11, %i12
+ %i14 = lshr i32 %i6, 4
+ %i15 = and i32 %i14, 4
+ %i16 = and i32 %i14, 6
+ %i17 = getelementptr inbounds i8, ptr %arg, i32 %i10
+ %i18 = load i8, ptr %i17, align 1, !tbaa !2
+ %i19 = zext i8 %i18 to i32
+ %i20 = lshr i32 %i19, %i15
+ %i21 = and i32 %i20, 15
+ %i22 = getelementptr inbounds i8, ptr %arg1, i32 %i13
+ %i23 = load i8, ptr %i22, align 1, !tbaa !2
+ %i24 = zext i8 %i23 to i32
+ %i25 = lshr i32 %i24, %i16
+ %i26 = shl nuw nsw i32 %i25, 4
+ %i27 = and i32 %i26, 48
+ %i28 = or disjoint i32 %i21, -32
+ %i29 = add nsw i32 %i28, %i27
+ %i30 = getelementptr inbounds i8, ptr %arg2, i32 %i14
+ %i31 = load i8, ptr %i30, align 1, !tbaa !2
+ %i32 = sext i8 %i31 to i32
+ %i33 = getelementptr inbounds i8, ptr %arg3, i32 %i6
+ %i34 = load i8, ptr %i33, align 1, !tbaa !2
+ %i35 = sext i8 %i34 to i32
+ %i36 = mul nsw i32 %i35, %i32
+ %i37 = mul nsw i32 %i36, %i29
+ %i38 = add nsw i32 %i37, %i
+ %i39 = add nuw nsw i32 %i6, 1
+ %i40 = icmp eq i32 %i39, 256
+ br i1 %i40, label %bb4, label %bb5, !llvm.loop !15
+}
+
+attributes #0 = { nofree norecurse nosync nounwind memory(argmem: read) "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+mutable-globals,+sign-ext,+simd128" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"Ubuntu clang version 18.1.8 (++20240731025043+3b5b5c1ec4a3-1~exp1~20240731145144.92)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = distinct !{!5, !6, !7}
+!6 = !{!"llvm.loop.mustprogress"}
+!7 = !{!"llvm.loop.unroll.disable"}
+!8 = distinct !{!8, !6, !7}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"short", !3, i64 0}
+!11 = distinct !{!11, !6, !7}
+!12 = distinct !{!12, !6, !7}
+!13 = distinct !{!13, !6, !7}
+!14 = distinct !{!14, !6, !7}
+!15 = distinct !{!15, !6, !7}
+;.
+; CHECK: [[CHAR_TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
+; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
+; CHECK: [[META6]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META7]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: [[META8]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META9]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META6]], [[META7]], [[META8]], [[META9]]}
+; CHECK: [[SHORT_TBAA11]] = !{[[META12:![0-9]+]], [[META12]], i64 0}
+; CHECK: [[META12]] = !{!"short", [[META3]], i64 0}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META6]], [[META7]], [[META8]], [[META9]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META6]], [[META7]], [[META8]], [[META9]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META6]], [[META7]], [[META8]], [[META9]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META6]], [[META7]], [[META8]], [[META9]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META6]], [[META7]], [[META8]], [[META9]]}
+;.
+; MAX: [[CHAR_TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
+; MAX: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; MAX: [[META4]] = !{!"Simple C/C++ TBAA"}
+; MAX: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
+; MAX: [[META6]] = !{!"llvm.loop.mustprogress"}
+; MAX: [[META7]] = !{!"llvm.loop.unroll.disable"}
+; MAX: [[META8]] = !{!"llvm.loop.isvectorized", i32 1}
+; MAX: [[META9]] = !{!"llvm.loop.unroll.runtime.disable"}
+; MAX: [[SHORT_TBAA10]] = !{[[META11:![0-9]+]], [[META11]], i64 0}
+; MAX: [[META11]] = !{!"short", [[META3]], i64 0}
+; MAX: [[LOOP12]] = distinct !{[[LOOP12]], [[META6]], [[META7]], [[META8]], [[META9]]}
+; MAX: [[LOOP13]] = distinct !{[[LOOP13]], [[META6]], [[META7]], [[META8]], [[META9]]}
+; MAX: [[LOOP14]] = distinct !{[[LOOP14]], [[META6]], [[META7]], [[META8]], [[META9]]}
+;.
More information about the llvm-commits
mailing list