[llvm] Extend narrow interleave group (PR #206305)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 27 22:13:05 PDT 2026
https://github.com/kamleshbhalui created https://github.com/llvm/llvm-project/pull/206305
1. Extend narrow interleave group to intrinsics.
2. Generalise the conditions on VF and vector register size
>From 4aaa97c54e4be16df2226b43b9d7a46f82343c03 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Fri, 26 Jun 2026 09:25:56 +0000
Subject: [PATCH 1/2] [Test] Add a new test and regenerate checks with script
for some old tests
---
.../CodeGen/WebAssembly/memory-interleave.ll | 3039 ++++++++++-------
.../AArch64/foldshuffleidentity.ll | 282 ++
.../LoopVectorize/AArch64/interleaved_cost.ll | 222 +-
.../AArch64/max-vf-for-interleaved.ll | 29 +-
.../LoopVectorize/ARM/interleaved_cost.ll | 136 +-
.../LoopVectorize/ARM/mve-interleaved-cost.ll | 1827 +++++++---
.../LoopVectorize/RISCV/interleaved-cost.ll | 1493 ++++++--
7 files changed, 4906 insertions(+), 2122 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll
diff --git a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
index 63cee30414a7c..2b7d53fa31d2b 100644
--- a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
+++ b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
@@ -1,4 +1,5 @@
-; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=loop-vectorize %s | llc -mtriple=wasm32 -mattr=+simd128 -asm-verbose=false -disable-wasm-fallthrough-return-opt | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "loop|v128\.|shuffle|i8x16|i16x8|i32x4|i64x2|f32x4|f64x2|i32\.(load|store)" --version 6
+; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=loop-vectorize %s | llc -mtriple=wasm32 -mattr=+simd128 -disable-wasm-fallthrough-return-opt | FileCheck %s
target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
@@ -18,23 +19,32 @@ target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20
%struct.TwoLongLongs = type { i64, i64 }
%struct.FourLongLongs = type { i64, i64, i64, i64 }
-; CHECK-LABEL: two_ints_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_ints_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: two_ints_same_op:
+; CHECK: loop # label2:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label3:
+; CHECK: i32.load 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: i32.load 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -62,23 +72,32 @@ define hidden void @two_ints_same_op(ptr noalias nocapture noundef writeonly %0,
br i1 %22, label %6, label %7
}
-; CHECK-LABEL: two_ints_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i32x4.sub
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_ints_vary_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: two_ints_vary_op:
+; CHECK: loop # label6:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i32x4.sub
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label7:
+; CHECK: i32.load 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: i32.load 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -106,22 +125,20 @@ define hidden void @two_ints_vary_op(ptr noalias nocapture noundef writeonly %0,
br i1 %22, label %6, label %7
}
+define hidden void @three_ints(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: three_ints:
-; CHECK: loop
+; CHECK: loop # label9:
; CHECK-NOT: v128.load
-; CHECK: i32.load
-; CHECK: i32.load
-; CHECK: i32.add
-; CHECK: i32.store
-; CHECK: i32.load
-; CHECK: i32.load
-; CHECK: i32.add
-; CHECK: i32.store
-; CHECK: i32.load
-; CHECK: i32.load
-; CHECK: i32.add
-; CHECK: i32.store
-define hidden void @three_ints(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK: i32.load 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: i32.load 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: i32.load 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -156,22 +173,20 @@ define hidden void @three_ints(ptr noalias nocapture noundef writeonly %0, ptr n
br i1 %28, label %6, label %7
}
+define hidden void @three_shorts(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: three_shorts:
-; CHECK: loop
+; CHECK: loop # label11:
; CHECK-NOT: v128.load
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.mul
-; CHECK: i32.store16
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.mul
-; CHECK: i32.store16
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.mul
-; CHECK: i32.store16
-define hidden void @three_shorts(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -206,33 +221,48 @@ define hidden void @three_shorts(ptr noalias nocapture noundef writeonly %0, ptr
br i1 %28, label %6, label %7
}
-; CHECK-LABEL: four_shorts_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_shorts_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_shorts_same_op:
+; CHECK: loop # label14:
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i16x8.sub
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i16x8.sub
+; CHECK: i8x16.shuffle 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i16x8.sub
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i16x8.sub
+; CHECK: i8x16.shuffle 0, 1, 0, 1, 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 0, 1, 0, 1, 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=1
+; CHECK: end_loop
+; CHECK: loop # label15:
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -274,33 +304,48 @@ define hidden void @four_shorts_same_op(ptr noalias nocapture noundef writeonly
br i1 %34, label %6, label %7
}
-; CHECK-LABEL: four_shorts_split_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.or
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.or
-; CHECK: i8x16.shuffle 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.xor
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.xor
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_shorts_split_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_shorts_split_op:
+; CHECK: loop # label18:
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.or
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.or
+; CHECK: i8x16.shuffle 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.xor
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.xor
+; CHECK: i8x16.shuffle 0, 1, 0, 1, 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 0, 1, 0, 1, 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=1
+; CHECK: end_loop
+; CHECK: loop # label19:
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -342,32 +387,48 @@ define hidden void @four_shorts_split_op(ptr noalias nocapture noundef writeonly
br i1 %34, label %6, label %7
}
-; CHECK-LABEL: four_shorts_interleave_op:
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.or
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.xor
-; CHECK: i8x16.shuffle 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.or
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.xor
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_shorts_interleave_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_shorts_interleave_op:
+; CHECK: loop # label22:
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.or
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.xor
+; CHECK: i8x16.shuffle 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.or
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.xor
+; CHECK: i8x16.shuffle 0, 1, 0, 1, 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19, 0, 1, 0, 1
+; CHECK: i8x16.shuffle 0, 1, 0, 1, 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=1
+; CHECK: end_loop
+; CHECK: loop # label23:
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.load16_u 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -409,30 +470,26 @@ define hidden void @four_shorts_interleave_op(ptr noalias nocapture noundef writ
br i1 %34, label %6, label %7
}
+define hidden void @five_shorts(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: five_shorts:
-; CHECK: loop
+; CHECK: loop # label25:
; CHECK-NOT: v128.load
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.sub
-; CHECK: i32.store16
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.sub
-; CHECK: i32.store16
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.sub
-; CHECK: i32.store16
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.sub
-; CHECK: i32.store16
-; CHECK: i32.load16_u
-; CHECK: i32.load16_u
-; CHECK: i32.sub
-; CHECK: i32.store16
-define hidden void @five_shorts(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.store16 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.store16 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.store16 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.store16 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.load16_u 0:p2align=0
+; CHECK: i32.store16 0:p2align=0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -481,25 +538,34 @@ define hidden void @five_shorts(ptr noalias nocapture noundef writeonly %0, ptr
br i1 %40, label %6, label %7
}
-; CHECK-LABEL: two_bytes_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
-; CHECK: i16x8.extmul_high_i8x16_u
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
-; CHECK: i16x8.extmul_high_i8x16_u
-; CHECK: i8x16.shuffle 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30
-; CHECK: v128.store
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30
-; CHECK: v128.store
define hidden void @two_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: two_bytes_same_op:
+; CHECK: loop # label28:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
+; CHECK: i16x8.extmul_high_i8x16_u
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
+; CHECK: i16x8.extmul_high_i8x16_u
+; CHECK: i8x16.shuffle 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30
+; CHECK: v128.store 16:p2align=0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label29:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -527,24 +593,33 @@ define hidden void @two_bytes_same_op(ptr noalias nocapture noundef writeonly %0
br i1 %22, label %6, label %7
}
-; CHECK-LABEL: two_bytes_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
-; CHECK: i16x8.extmul_high_i8x16_u
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
-; CHECK: i8x16.sub
-; CHECK: i8x16.shuffle 0, 24, 2, 25, 4, 26, 6, 27, 8, 28, 10, 29, 12, 30, 14, 31
-; CHECK: v128.store
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 16, 2, 17, 4, 18, 6, 19, 8, 20, 10, 21, 12, 22, 14, 23
-; CHECK: v128.store
define hidden void @two_bytes_vary_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: two_bytes_vary_op:
+; CHECK: loop # label32:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
+; CHECK: i16x8.extmul_high_i8x16_u
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
+; CHECK: i8x16.sub
+; CHECK: i8x16.shuffle 0, 24, 2, 25, 4, 26, 6, 27, 8, 28, 10, 29, 12, 30, 14, 31
+; CHECK: v128.store 16:p2align=0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 16, 2, 17, 4, 18, 6, 19, 8, 20, 10, 21, 12, 22, 14, 23
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label33:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -572,21 +647,19 @@ define hidden void @two_bytes_vary_op(ptr noalias nocapture noundef writeonly %0
br i1 %22, label %6, label %7
}
-; CHECK-LABEL: three_bytes_same_op:
-; CHECK: loop
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.and
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.and
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.and
-; CHECK: i32.store8
define hidden void @three_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: three_bytes_same_op:
+; CHECK: loop # label35:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -621,21 +694,19 @@ define hidden void @three_bytes_same_op(ptr noalias nocapture noundef writeonly
br i1 %28, label %6, label %7
}
-; CHECK-LABEL: three_bytes_interleave_op:
-; CHECK: loop
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
define hidden void @three_bytes_interleave_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: three_bytes_interleave_op:
+; CHECK: loop # label37:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -670,33 +741,48 @@ define hidden void @three_bytes_interleave_op(ptr noalias nocapture noundef writ
br i1 %28, label %6, label %7
}
-; CHECK-LABEL: four_bytes_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 0, 0, 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23
-; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19
-; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
-; CHECK: v128.store
define hidden void @four_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_bytes_same_op:
+; CHECK: loop # label40:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.and
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.and
+; CHECK: i8x16.shuffle 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23, 0, 0
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.and
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.and
+; CHECK: i8x16.shuffle 0, 0, 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23
+; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
+; CHECK: v128.store 16:p2align=0
+; CHECK: i8x16.shuffle 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19
+; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label41:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -738,31 +824,46 @@ define hidden void @four_bytes_same_op(ptr noalias nocapture noundef writeonly %
br i1 %34, label %6, label %7
}
-; CHECK-LABEL: four_bytes_split_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.sub
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.sub
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27
-; CHECK: v128.store
define hidden void @four_bytes_split_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_bytes_split_op:
+; CHECK: loop # label44:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.sub
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.sub
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
+; CHECK: v128.store 16:p2align=0
+; CHECK: i8x16.shuffle 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label45:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -804,33 +905,48 @@ define hidden void @four_bytes_split_op(ptr noalias nocapture noundef writeonly
br i1 %34, label %6, label %7
}
-; CHECK-LABEL: four_bytes_interleave_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.add
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.sub
-; CHECK: i8x16.shuffle 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.add
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.sub
-; CHECK: i8x16.shuffle 0, 0, 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23
-; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19
-; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
-; CHECK: v128.store
define hidden void @four_bytes_interleave_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_bytes_interleave_op:
+; CHECK: loop # label48:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.sub
+; CHECK: i8x16.shuffle 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23, 0, 0
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.sub
+; CHECK: i8x16.shuffle 0, 0, 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23
+; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
+; CHECK: v128.store 16:p2align=0
+; CHECK: i8x16.shuffle 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19
+; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label49:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -872,42 +988,35 @@ define hidden void @four_bytes_interleave_op(ptr noalias nocapture noundef write
br i1 %34, label %6, label %7
}
+define hidden void @eight_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: eight_bytes_same_op:
-; CHECK: loop
+; CHECK: loop # label51:
; CHECK-NOT: v128.load
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.store8
-define hidden void @eight_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -977,42 +1086,35 @@ define hidden void @eight_bytes_same_op(ptr noalias nocapture noundef writeonly
br i1 %58, label %6, label %7
}
+define hidden void @eight_bytes_split_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: eight_bytes_split_op:
-; CHECK: loop
+; CHECK: loop # label53:
; CHECK-NOT: v128.load
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-define hidden void @eight_bytes_split_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -1082,42 +1184,35 @@ define hidden void @eight_bytes_split_op(ptr noalias nocapture noundef writeonly
br i1 %58, label %6, label %7
}
+define hidden void @eight_bytes_interleave_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: eight_bytes_interleave_op:
-; CHECK: loop
+; CHECK: loop # label55:
; CHECK-NOT: v128.load
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.add
-; CHECK: i32.store8
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.sub
-; CHECK: i32.store8
-define hidden void @eight_bytes_interleave_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -1187,33 +1282,86 @@ define hidden void @eight_bytes_interleave_op(ptr noalias nocapture noundef writ
br i1 %58, label %6, label %7
}
-; CHECK-LABEL: four_bytes_into_four_ints_same_op:
-; CHECK: loop
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.load
-; CHECK: i32.add
-; CHECK: i32.store
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.load
-; CHECK: i32.add
-; CHECK: i32.store
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.load
-; CHECK: i32.add
-; CHECK: i32.store
-; CHECK: i32.load8_u
-; CHECK: i32.load8_u
-; CHECK: i32.mul
-; CHECK: i32.load
-; CHECK: i32.add
-; CHECK: i32.store
define hidden void @four_bytes_into_four_ints_same_op(ptr noalias nocapture noundef %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_bytes_into_four_ints_same_op:
+; CHECK: loop # label58:
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extmul_low_i16x8_u
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: v128.load 32:p2align=2
+; CHECK: v128.load 48:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extmul_low_i16x8_u
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extmul_low_i16x8_u
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extmul_low_i16x8_u
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 32:p2align=2
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 48:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label59:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load 0
+; CHECK: i32.store 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -1271,50 +1419,65 @@ define hidden void @four_bytes_into_four_ints_same_op(ptr noalias nocapture noun
br i1 %50, label %6, label %7
}
-; CHECK-LABEL: four_bytes_into_four_ints_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extend_low_i16x8_u
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extend_low_i16x8_u
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extend_low_i16x8_u
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extend_low_i16x8_u
-; CHECK: i32x4.sub
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extmul_low_i16x8_u
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extend_low_i16x8_u
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_bytes_into_four_ints_vary_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
+; CHECK-LABEL: four_bytes_into_four_ints_vary_op:
+; CHECK: loop # label62:
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extend_low_i16x8_u
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extend_low_i16x8_u
+; CHECK: i32x4.add
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extend_low_i16x8_u
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extend_low_i16x8_u
+; CHECK: i32x4.sub
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extmul_low_i16x8_u
+; CHECK: v128.and
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i32x4.extend_low_i16x8_u
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 48:p2align=2
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 32:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label63:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store 0
+; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -1363,13 +1526,20 @@ define hidden void @four_bytes_into_four_ints_vary_op(ptr noalias nocapture noun
br i1 %41, label %6, label %7
}
-; CHECK-LABEL: scale_uv_row_down2:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29
-; CHECK: v128.store
define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
+; CHECK-LABEL: scale_uv_row_down2:
+; CHECK: loop # label67:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label68:
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp sgt i32 %3, 0
br i1 %5, label %6, label %19
@@ -1394,41 +1564,55 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
ret void
}
-; CHECK-LABEL: scale_uv_row_down2_box:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i8x16.shuffle 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30
-; CHECK: v128.store
define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
+; CHECK-LABEL: scale_uv_row_down2_box:
+; CHECK: loop # label72:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: v128.const 2, 2, 2, 2, 2, 2, 2, 2
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i8x16.shuffle 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label73:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp sgt i32 %3, 0
br i1 %5, label %6, label %54
@@ -1490,19 +1674,28 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
ret void
}
-; CHECK-LABEL: scale_uv_row_down2_linear:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.avgr_u
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.avgr_u
-; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
-; CHECK: v128.store
define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
+; CHECK-LABEL: scale_uv_row_down2_linear:
+; CHECK: loop # label78:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.avgr_u
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.avgr_u
+; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label79:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
%5 = icmp sgt i32 %3, 0
br i1 %5, label %6, label %34
@@ -1542,137 +1735,161 @@ define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0,
ret void
}
-; CHECK-LABEL: scale_argb_row_down2_box:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i8x16.shuffle 8, 24, 0, 0, 10, 26, 0, 0, 12, 28, 0, 0, 14, 30, 0, 0
-; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i16x8.extract_lane_u 4
-; CHECK: i8x16.replace_lane 2
-; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i16x8.extract_lane_u 4
-; CHECK: i8x16.replace_lane 3
-; CHECK: i16x8.extract_lane_u 5
-; CHECK: i8x16.replace_lane 6
-; CHECK: i16x8.extract_lane_u 5
-; CHECK: i8x16.replace_lane 7
-; CHECK: i16x8.extract_lane_u 6
-; CHECK: i8x16.replace_lane 10
-; CHECK: i16x8.extract_lane_u 6
-; CHECK: i8x16.replace_lane 11
-; CHECK: i16x8.extract_lane_u 7
-; CHECK: i8x16.replace_lane 14
-; CHECK: i16x8.extract_lane_u 7
-; CHECK: i8x16.replace_lane 15
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 16, 0, 0, 2, 18, 0, 0, 4, 20, 0, 0, 6, 22, 0, 0
-; CHECK: i16x8.extract_lane_u 0
-; CHECK: i8x16.replace_lane 2
-; CHECK: i16x8.extract_lane_u 0
-; CHECK: i8x16.replace_lane 3
-; CHECK: i16x8.extract_lane_u 1
-; CHECK: i8x16.replace_lane 6
-; CHECK: i16x8.extract_lane_u 1
-; CHECK: i8x16.replace_lane 7
-; CHECK: i16x8.extract_lane_u 2
-; CHECK: i8x16.replace_lane 10
-; CHECK: i16x8.extract_lane_u 2
-; CHECK: i8x16.replace_lane 11
-; CHECK: i16x8.extract_lane_u 3
-; CHECK: i8x16.replace_lane 14
-; CHECK: i16x8.extract_lane_u 3
-; CHECK: i8x16.replace_lane 15
-; CHECK: v128.store
define hidden void @scale_argb_row_down2_box(ptr noundef readonly captures(none) %arg, i32 noundef %arg1, ptr noundef writeonly captures(none) %arg2, i32 noundef %arg3) local_unnamed_addr #0 {
+; CHECK-LABEL: scale_argb_row_down2_box:
+; CHECK: loop # label83:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 32:p2align=0
+; CHECK: v128.load 48:p2align=0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 32:p2align=0
+; CHECK: v128.load 48:p2align=0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: v128.const 2, 2, 2, 2, 2, 2, 2, 2
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i8x16.shuffle 8, 24, 0, 0, 10, 26, 0, 0, 12, 28, 0, 0, 14, 30, 0, 0
+; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i16x8.extract_lane_u 4
+; CHECK: i8x16.replace_lane 2
+; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i16x8.extract_lane_u 4
+; CHECK: i8x16.replace_lane 3
+; CHECK: i16x8.extract_lane_u 5
+; CHECK: i8x16.replace_lane 6
+; CHECK: i16x8.extract_lane_u 5
+; CHECK: i8x16.replace_lane 7
+; CHECK: i16x8.extract_lane_u 6
+; CHECK: i8x16.replace_lane 10
+; CHECK: i16x8.extract_lane_u 6
+; CHECK: i8x16.replace_lane 11
+; CHECK: i16x8.extract_lane_u 7
+; CHECK: i8x16.replace_lane 14
+; CHECK: i16x8.extract_lane_u 7
+; CHECK: i8x16.replace_lane 15
+; CHECK: v128.store 16:p2align=0
+; CHECK: i8x16.shuffle 0, 16, 0, 0, 2, 18, 0, 0, 4, 20, 0, 0, 6, 22, 0, 0
+; CHECK: i16x8.extract_lane_u 0
+; CHECK: i8x16.replace_lane 2
+; CHECK: i16x8.extract_lane_u 0
+; CHECK: i8x16.replace_lane 3
+; CHECK: i16x8.extract_lane_u 1
+; CHECK: i8x16.replace_lane 6
+; CHECK: i16x8.extract_lane_u 1
+; CHECK: i8x16.replace_lane 7
+; CHECK: i16x8.extract_lane_u 2
+; CHECK: i8x16.replace_lane 10
+; CHECK: i16x8.extract_lane_u 2
+; CHECK: i8x16.replace_lane 11
+; CHECK: i16x8.extract_lane_u 3
+; CHECK: i8x16.replace_lane 14
+; CHECK: i16x8.extract_lane_u 3
+; CHECK: i8x16.replace_lane 15
+; CHECK: v128.store 0:p2align=0
+; CHECK: end_loop
+; CHECK: loop # label84:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
bb:
%i = icmp sgt i32 %arg3, 0
br i1 %i, label %bb4, label %bb86
@@ -1769,125 +1986,173 @@ bb86: ; preds = %bb4, %bb
ret void
}
-; CHECK-LABEL: argb_to_uv_matrix_row
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: v128.const 2, 2, 2, 2, 2, 2, 2, 2
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i16x8.mul
-; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i16x8.mul
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i16x8.mul
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i16x8.add
-; CHECK: i16x8.add
-; CHECK: i16x8.shr_u
-; CHECK: i16x8.mul
-; CHECK: i16x8.add
-; CHECK: i16x8.sub
-; CHECK: i16x8.shr_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.store64_lane
-; CHECK: i16x8.mul
-; CHECK: i16x8.mul
-; CHECK: i16x8.add
-; CHECK: i16x8.mul
-; CHECK: i16x8.add
-; CHECK: i16x8.mul
-; CHECK: i16x8.add
-; CHECK: i16x8.sub
-; CHECK: i16x8.shr_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.store64_lane
; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite)
define hidden void @argb_to_uv_matrix_row(ptr noundef readonly captures(none) %arg, i32 noundef %arg1, ptr noundef writeonly captures(none) %arg2, ptr noundef writeonly captures(none) %arg3, i32 noundef %arg4, ptr noundef readonly captures(none) %arg5) local_unnamed_addr {
+; CHECK-LABEL: argb_to_uv_matrix_row:
+; CHECK: v128.load8_splat 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: v128.load16_splat 128
+; CHECK: v128.load8_splat 67
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: v128.load8_splat 66
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: v128.load8_splat 65
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: v128.load8_splat 64
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: v128.load8_splat 35
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: v128.load8_splat 34
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: v128.load8_splat 33
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: loop # label88:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 32:p2align=0
+; CHECK: v128.load 48:p2align=0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 32:p2align=0
+; CHECK: v128.load 48:p2align=0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 5, 13, 21, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: v128.const 2, 2, 2, 2, 2, 2, 2, 2
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i16x8.mul
+; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i16x8.mul
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 2, 10, 18, 26, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i16x8.mul
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 3, 11, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i8x16.shuffle 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 0, 0, 0, 7, 15, 23, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_u
+; CHECK: i16x8.add
+; CHECK: i16x8.add
+; CHECK: i16x8.shr_u
+; CHECK: i16x8.mul
+; CHECK: i16x8.add
+; CHECK: i16x8.sub
+; CHECK: i16x8.shr_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.store64_lane 0:p2align=0, 0
+; CHECK: i16x8.mul
+; CHECK: i16x8.mul
+; CHECK: i16x8.add
+; CHECK: i16x8.mul
+; CHECK: i16x8.add
+; CHECK: i16x8.mul
+; CHECK: i16x8.add
+; CHECK: i16x8.sub
+; CHECK: i16x8.shr_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.store64_lane 0:p2align=0, 0
+; CHECK: end_loop
+; CHECK: loop # label89:
+; CHECK: i32.load16_u 128
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_s 33
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_s 34
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_s 35
+; CHECK: i32.store8 0
+; CHECK: i32.load16_u 128
+; CHECK: i32.load8_s 65
+; CHECK: i32.load8_s 64
+; CHECK: i32.load8_s 66
+; CHECK: i32.load8_s 67
+; CHECK: i32.store8 0
+; CHECK: end_loop
bb:
%i = getelementptr inbounds i8, ptr %arg, i32 %arg1
%i6 = add nsw i32 %arg4, -1
@@ -2030,23 +2295,26 @@ bb134: ; preds = %bb18, %bb
ret void
}
-; CHECK-LABEL: two_floats_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_floats_same_op:
+; CHECK: loop # label92:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label93:
+; CHECK: end_loop
entry:
%cmp21.not = icmp eq i32 %N, 0
br i1 %cmp21.not, label %for.cond.cleanup, label %for.body
@@ -2075,23 +2343,26 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_floats_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: f32x4.add
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.sub
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_floats_vary_op:
+; CHECK: loop # label96:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: f32x4.add
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: f32x4.sub
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label97:
+; CHECK: end_loop
entry:
%cmp20.not = icmp eq i32 %N, 0
br i1 %cmp20.not, label %for.cond.cleanup, label %for.body
@@ -2120,33 +2391,40 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_bytes_two_floats_same_op:
-; CHECK: loop
-; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_bytes_two_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_bytes_two_floats_same_op:
+; CHECK: loop # label100:
+; CHECK: v128.load64_zero 0:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load64_zero 0:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label101:
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: end_loop
entry:
%cmp24.not = icmp eq i32 %N, 0
br i1 %cmp24.not, label %for.cond.cleanup, label %for.body
@@ -2179,32 +2457,40 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_bytes_two_floats_vary_op:
-; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.add
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.sub
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_bytes_two_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_bytes_two_floats_vary_op:
+; CHECK: loop # label104:
+; CHECK: v128.load64_zero 0:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load64_zero 0:p2align=0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.add
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.sub
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label105:
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: end_loop
entry:
%cmp23.not = icmp eq i32 %N, 0
br i1 %cmp23.not, label %for.cond.cleanup, label %for.body
@@ -2237,30 +2523,36 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_floats_two_bytes_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: f32x4.mul
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.const 255, 255, 255, 255
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.narrow_i16x8_u
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.mul
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.narrow_i16x8_u
-; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.store64_lane
define hidden void @two_floats_two_bytes_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_floats_two_bytes_same_op:
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: loop # label108:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: f32x4.mul
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.const 255, 255, 255, 255
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.narrow_i16x8_u
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: f32x4.mul
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.narrow_i16x8_u
+; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.store64_lane 0:p2align=0, 0
+; CHECK: end_loop
+; CHECK: loop # label109:
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
entry:
%cmp22.not = icmp eq i32 %N, 0
br i1 %cmp22.not, label %for.cond.cleanup, label %for.body
@@ -2291,30 +2583,36 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_floats_two_bytes_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: f32x4.add
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.const 255, 255, 255, 255
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.narrow_i16x8_u
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.sub
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.narrow_i16x8_u
-; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.store64_lane
define hidden void @two_floats_two_bytes_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_floats_two_bytes_vary_op:
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: loop # label112:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: f32x4.add
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.const 255, 255, 255, 255
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.narrow_i16x8_u
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: f32x4.sub
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.narrow_i16x8_u
+; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.store64_lane 0:p2align=0, 0
+; CHECK: end_loop
+; CHECK: loop # label113:
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
entry:
%cmp21.not = icmp eq i32 %N, 0
br i1 %cmp21.not, label %for.cond.cleanup, label %for.body
@@ -2345,29 +2643,36 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_shorts_two_floats_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_shorts_two_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_shorts_two_floats_same_op:
+; CHECK: loop # label116:
+; CHECK: v128.load 0:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load 0:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label117:
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: end_loop
entry:
%cmp24.not = icmp eq i32 %N, 0
br i1 %cmp24.not, label %for.cond.cleanup, label %for.body
@@ -2400,29 +2705,36 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_shorts_two_floats_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.add
-; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.sub
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store
define hidden void @two_shorts_two_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_shorts_two_floats_vary_op:
+; CHECK: loop # label120:
+; CHECK: v128.load 0:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load 0:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.add
+; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.sub
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label121:
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: end_loop
entry:
%cmp23.not = icmp eq i32 %N, 0
br i1 %cmp23.not, label %for.cond.cleanup, label %for.body
@@ -2455,28 +2767,33 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_floats_two_shorts_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: f32x4.mul
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.const 65535, 65535, 65535, 65535
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.mul
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.shuffle 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
-; CHECK: v128.store
define hidden void @two_floats_two_shorts_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_floats_two_shorts_same_op:
+; CHECK: loop # label124:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: f32x4.mul
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.const 65535, 65535, 65535, 65535
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: f32x4.mul
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.shuffle 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
+; CHECK: v128.store 0:p2align=1
+; CHECK: end_loop
+; CHECK: loop # label125:
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
entry:
%cmp22.not = icmp eq i32 %N, 0
br i1 %cmp22.not, label %for.cond.cleanup, label %for.body
@@ -2507,28 +2824,33 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: two_floats_two_shorts_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: f32x4.add
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.const 65535, 65535, 65535, 65535
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.sub
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.shuffle 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
-; CHECK: v128.store
define hidden void @two_floats_two_shorts_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: two_floats_two_shorts_vary_op:
+; CHECK: loop # label128:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: f32x4.add
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.const 65535, 65535, 65535, 65535
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+; CHECK: f32x4.sub
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.shuffle 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
+; CHECK: v128.store 0:p2align=1
+; CHECK: end_loop
+; CHECK: loop # label129:
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
entry:
%cmp21.not = icmp eq i32 %N, 0
br i1 %cmp21.not, label %for.cond.cleanup, label %for.body
@@ -2559,12 +2881,16 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: four_floats_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: v128.store
define hidden void @four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: four_floats_same_op:
+; CHECK: loop # label132:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 0:p2align=2
+; CHECK: f32x4.mul
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label133:
+; CHECK: end_loop
entry:
%cmp45.not = icmp eq i32 %N, 0
br i1 %cmp45.not, label %for.cond.cleanup, label %for.body
@@ -2607,9 +2933,11 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+define hidden void @four_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_vary_op:
; CHECK-NOT: f32x4
-define hidden void @four_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK: loop # label135:
+; CHECK: end_loop
entry:
%cmp42.not = icmp eq i32 %N, 0
br i1 %cmp42.not, label %for.cond.cleanup, label %for.body
@@ -2652,63 +2980,74 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: four_bytes_four_floats_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_bytes_four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: four_bytes_four_floats_same_op:
+; CHECK: loop # label138:
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 48:p2align=2
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 32:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label139:
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: end_loop
entry:
%cmp52.not = icmp eq i32 %N, 0
br i1 %cmp52.not, label %for.cond.cleanup, label %for.body
@@ -2759,63 +3098,74 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: four_bytes_four_floats_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.add
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.div
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.sub
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_bytes_four_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: four_bytes_four_floats_vary_op:
+; CHECK: loop # label142:
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load 0:p2align=0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.add
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.div
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extend_low_i8x16_s
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.sub
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 48:p2align=2
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 32:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label143:
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: i32.load8_s 0
+; CHECK: end_loop
entry:
%cmp49.not = icmp eq i32 %N, 0
br i1 %cmp49.not, label %for.cond.cleanup, label %for.body
@@ -2866,9 +3216,15 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+define hidden void @four_floats_four_bytes_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_bytes_same_op:
; CHECK-NOT: v128.load
-define hidden void @four_floats_four_bytes_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK: loop # label145:
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
entry:
%cmp48.not = icmp eq i32 %N, 0
br i1 %cmp48.not, label %for.cond.cleanup, label %for.body
@@ -2915,9 +3271,15 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+define hidden void @four_floats_four_bytes_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_bytes_vary_op:
; CHECK-NOT: v128.load
-define hidden void @four_floats_four_bytes_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK: loop # label147:
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: i32.store8 0
+; CHECK: end_loop
entry:
%cmp45.not = icmp eq i32 %N, 0
br i1 %cmp45.not, label %for.cond.cleanup, label %for.body
@@ -2964,57 +3326,68 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: four_shorts_four_floats_same_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_shorts_four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: four_shorts_four_floats_same_op:
+; CHECK: loop # label150:
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 48:p2align=2
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 32:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label151:
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: end_loop
entry:
%cmp52.not = icmp eq i32 %N, 0
br i1 %cmp52.not, label %for.cond.cleanup, label %for.body
@@ -3065,53 +3438,68 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-LABEL: four_shorts_four_floats_vary_op:
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.add
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.div
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.sub
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store
define hidden void @four_shorts_four_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK-LABEL: four_shorts_four_floats_vary_op:
+; CHECK: loop # label154:
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: v128.load 0:p2align=1
+; CHECK: v128.load 16:p2align=1
+; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.mul
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.add
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.div
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: f32x4.convert_i32x4_s
+; CHECK: f32x4.sub
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 48:p2align=2
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 32:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 16:p2align=2
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
+; CHECK: v128.store 0:p2align=2
+; CHECK: end_loop
+; CHECK: loop # label155:
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: i32.load16_s 0
+; CHECK: end_loop
entry:
%cmp49.not = icmp eq i32 %N, 0
br i1 %cmp49.not, label %for.cond.cleanup, label %for.body
@@ -3162,9 +3550,15 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+define hidden void @four_floats_four_shorts_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_shorts_same_op:
; CHECK-NOT: v128.load
-define hidden void @four_floats_four_shorts_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK: loop # label157:
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
entry:
%cmp48.not = icmp eq i32 %N, 0
br i1 %cmp48.not, label %for.cond.cleanup, label %for.body
@@ -3211,9 +3605,15 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+define hidden void @four_floats_four_shorts_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_shorts_vary_op:
; CHECK-NOT: v128.load
-define hidden void @four_floats_four_shorts_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
+; CHECK: loop # label159:
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: i32.store16 0
+; CHECK: end_loop
entry:
%cmp45.not = icmp eq i32 %N, 0
br i1 %cmp45.not, label %for.cond.cleanup, label %for.body
@@ -3260,8 +3660,20 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-; CHECK-NOT: v128.load
define hidden void @mac_3d_i8(ptr dead_on_unwind noalias writable writeonly sret(%struct.ThreeBytes) align 1 captures(none) %0, ptr noundef readonly captures(none) %1, ptr noundef readonly captures(none) %2, i32 noundef %3) {
+; CHECK-LABEL: mac_3d_i8:
+; CHECK: loop # label162:
+; CHECK-NOT: v128.load
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: end_loop
+; CHECK: i32.store8 2
+; CHECK: i32.store8 1
+; CHECK: i32.store8 0
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %12
@@ -3304,33 +3716,78 @@ define hidden void @mac_3d_i8(ptr dead_on_unwind noalias writable writeonly sret
br i1 %36, label %6, label %12
}
-; CHECK-LABEL: mac_4d_i8
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.add
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.add
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.add
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.add
define hidden void @mac_4d_i8(ptr dead_on_unwind noalias writable writeonly sret(%struct.FourBytes) align 1 captures(none) initializes((0, 4)) %0, ptr noundef readonly captures(none) %1, ptr noundef readonly captures(none) %2, i32 noundef %3) {
+; CHECK-LABEL: mac_4d_i8:
+; CHECK: i32.store 0:p2align=0
+; CHECK: v128.const 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: loop # label166:
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load 0:p2align=0
+; CHECK: v128.load 16:p2align=0
+; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: end_loop
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.extract_lane_u 0
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.extract_lane_u 0
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.extract_lane_u 0
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.shuffle 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.add
+; CHECK: i8x16.extract_lane_u 0
+; CHECK: loop # label167:
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: i32.load8_u 0
+; CHECK: end_loop
+; CHECK: i32.store8 3
+; CHECK: i32.store8 2
+; CHECK: i32.store8 1
+; CHECK: i32.store8 0
store i32 0, ptr %0, align 1
%5 = icmp eq i32 %3, 0
br i1 %5, label %11, label %6
@@ -3387,23 +3844,35 @@ define hidden void @mac_4d_i8(ptr dead_on_unwind noalias writable writeonly sret
br i1 %44, label %10, label %13
}
-; CHECK-LABEL: mac_2d_i64
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_u
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_s
-; CHECK: i64x2.mul
-; CHECK: i64x2.add
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_u
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_s
-; CHECK: i64x2.mul
-; CHECK: i64x2.add
define hidden void @mac_2d_i64(ptr dead_on_unwind noalias writable sret(%struct.TwoLongLongs) align 8 captures(none) %agg.result, ptr noundef readonly captures(none) %x, ptr noundef readonly captures(none) %y, i32 noundef %n) {
+; CHECK-LABEL: mac_2d_i64:
+; CHECK: v128.const 0, 0
+; CHECK: i64x2.replace_lane 0
+; CHECK: i64x2.replace_lane 0
+; CHECK: loop # label171:
+; CHECK: v128.load 0:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_u
+; CHECK: v128.load 0:p2align=2
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_s
+; CHECK: i64x2.mul
+; CHECK: i64x2.add
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_u
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_s
+; CHECK: i64x2.mul
+; CHECK: i64x2.add
+; CHECK: end_loop
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
+; CHECK: i64x2.add
+; CHECK: i64x2.extract_lane 0
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
+; CHECK: i64x2.add
+; CHECK: i64x2.extract_lane 0
+; CHECK: loop # label172:
+; CHECK: end_loop
entry:
%agg.result.promoted = load i64, ptr %agg.result, align 8
%cmp20.not = icmp eq i32 %n, 0
@@ -3448,37 +3917,57 @@ for.body: ; preds = %for.body.lr.ph, %fo
br i1 %exitcond.not, label %for.cond.for.cond.cleanup_crit_edge, label %for.body
}
-; CHECK-LABEL: mac_4d_i64
-; CHECK: loop
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_u
-; CHECK: v128.load
-; CHECK: v128.load
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_s
-; CHECK: i64x2.mul
-; CHECK: i64x2.add
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_u
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_s
-; CHECK: i64x2.mul
-; CHECK: i64x2.add
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_u
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_s
-; CHECK: i64x2.mul
-; CHECK: i64x2.add
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_u
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i64x2.extend_low_i32x4_s
-; CHECK: i64x2.mul
-; CHECK: i64x2.add
define hidden void @mac_4d_i64(ptr dead_on_unwind noalias writable sret(%struct.FourLongLongs) align 8 captures(none) %agg.result, ptr noundef readonly captures(none) %x, ptr noundef readonly captures(none) %y, i32 noundef %n) {
+; CHECK-LABEL: mac_4d_i64:
+; CHECK: v128.const 0, 0
+; CHECK: i64x2.replace_lane 0
+; CHECK: i64x2.replace_lane 0
+; CHECK: i64x2.replace_lane 0
+; CHECK: i64x2.replace_lane 0
+; CHECK: loop # label176:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_u
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 16:p2align=2
+; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_s
+; CHECK: i64x2.mul
+; CHECK: i64x2.add
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_u
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_s
+; CHECK: i64x2.mul
+; CHECK: i64x2.add
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_u
+; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_s
+; CHECK: i64x2.mul
+; CHECK: i64x2.add
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_u
+; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
+; CHECK: i64x2.extend_low_i32x4_s
+; CHECK: i64x2.mul
+; CHECK: i64x2.add
+; CHECK: end_loop
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
+; CHECK: i64x2.add
+; CHECK: i64x2.extract_lane 0
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
+; CHECK: i64x2.add
+; CHECK: i64x2.extract_lane 0
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
+; CHECK: i64x2.add
+; CHECK: i64x2.extract_lane 0
+; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
+; CHECK: i64x2.add
+; CHECK: i64x2.extract_lane 0
+; CHECK: loop # label177:
+; CHECK: end_loop
entry:
%agg.result.promoted = load i64, ptr %agg.result, align 8
%cmp44.not = icmp eq i32 %n, 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll b/llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll
new file mode 100644
index 0000000000000..0aa9c976ac508
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll
@@ -0,0 +1,282 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -p loop-vectorize < %s | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @f64_to_q15(ptr nofree noundef readonly captures(none) %pSrc, ptr nofree noundef writeonly captures(none) %pDst, i32 noundef %blockSize) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @f64_to_q15(
+; CHECK-SAME: ptr nofree noundef readonly captures(none) [[PSRC:%.*]], ptr nofree noundef writeonly captures(none) [[PDST:%.*]], i32 noundef [[BLOCKSIZE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[BLOCKSIZE]], 2
+; CHECK-NEXT: [[CMP_NOT65:%.*]] = icmp eq i32 [[SHR]], 0
+; CHECK-NEXT: br i1 [[CMP_NOT65]], label %[[WHILE_END:.*]], label %[[WHILE_BODY_PREHEADER:.*]]
+; CHECK: [[WHILE_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP34:%.*]] = zext i32 [[BLOCKSIZE]] to i64
+; CHECK-NEXT: [[TMP35:%.*]] = lshr i64 [[TMP34]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP35]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP35]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP35]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP36:%.*]] = shl i64 [[N_VEC]], 5
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[PSRC]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP39:%.*]] = sub i32 [[SHR]], [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr i8, ptr [[PDST]], i64 [[TMP40]]
+; CHECK-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT: [[TMP42:%.*]] = shl i64 [[INDEX]], 5
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC]], i64 [[TMP42]]
+; CHECK-NEXT: [[TMP43:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PDST]], i64 [[TMP43]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x double>, ptr [[NEXT_GEP]], align 8, !tbaa [[DOUBLE_TBAA7:![0-9]+]]
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[TMP10:%.*]] = fmul <8 x double> [[STRIDED_VEC]], splat (double 3.276800e+04)
+; CHECK-NEXT: [[TMP11:%.*]] = fptosi <8 x double> [[TMP10]] to <8 x i32>
+; CHECK-NEXT: [[TMP12:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP11]], <8 x i32> splat (i32 -32768))
+; CHECK-NEXT: [[TMP13:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP12]], <8 x i32> splat (i32 32767))
+; CHECK-NEXT: [[TMP14:%.*]] = trunc nsw <8 x i32> [[TMP13]] to <8 x i16>
+; CHECK-NEXT: [[TMP15:%.*]] = fmul <8 x double> [[STRIDED_VEC2]], splat (double 3.276800e+04)
+; CHECK-NEXT: [[TMP16:%.*]] = fptosi <8 x double> [[TMP15]] to <8 x i32>
+; CHECK-NEXT: [[TMP17:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP16]], <8 x i32> splat (i32 -32768))
+; CHECK-NEXT: [[TMP18:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP17]], <8 x i32> splat (i32 32767))
+; CHECK-NEXT: [[TMP19:%.*]] = trunc nsw <8 x i32> [[TMP18]] to <8 x i16>
+; CHECK-NEXT: [[TMP20:%.*]] = fmul <8 x double> [[STRIDED_VEC3]], splat (double 3.276800e+04)
+; CHECK-NEXT: [[TMP21:%.*]] = fptosi <8 x double> [[TMP20]] to <8 x i32>
+; CHECK-NEXT: [[TMP22:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP21]], <8 x i32> splat (i32 -32768))
+; CHECK-NEXT: [[TMP23:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP22]], <8 x i32> splat (i32 32767))
+; CHECK-NEXT: [[TMP24:%.*]] = trunc nsw <8 x i32> [[TMP23]] to <8 x i16>
+; CHECK-NEXT: [[TMP25:%.*]] = fmul <8 x double> [[STRIDED_VEC4]], splat (double 3.276800e+04)
+; CHECK-NEXT: [[TMP26:%.*]] = fptosi <8 x double> [[TMP25]] to <8 x i32>
+; CHECK-NEXT: [[TMP27:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP26]], <8 x i32> splat (i32 -32768))
+; CHECK-NEXT: [[TMP28:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP27]], <8 x i32> splat (i32 32767))
+; CHECK-NEXT: [[TMP29:%.*]] = trunc nsw <8 x i32> [[TMP28]] to <8 x i16>
+; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <8 x i16> [[TMP14]], <8 x i16> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <8 x i16> [[TMP24]], <8 x i16> [[TMP29]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <16 x i16> [[TMP30]], <16 x i16> [[TMP31]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i16> [[TMP32]], <32 x i16> poison, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP1]], align 2, !tbaa [[SHORT_TBAA9:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[WHILE_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP35]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP37]], %[[MIDDLE_BLOCK]] ], [ [[PSRC]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP39]], %[[MIDDLE_BLOCK]] ], [ [[SHR]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[PDST]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[WHILE_BODY1:.*]]
+; CHECK: [[WHILE_BODY1]]:
+; CHECK-NEXT: [[PIN_068:%.*]] = phi ptr [ [[INCDEC_PTR15:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BLKCNT_067:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[PDST_ADDR_066:%.*]] = phi ptr [ [[INCDEC_PTR20:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PIN_068]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[PIN_068]], align 8, !tbaa [[DOUBLE_TBAA7]]
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[TMP0]], 3.276800e+04
+; CHECK-NEXT: [[CONV:%.*]] = fptosi double [[MUL]] to i32
+; CHECK-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.smax.i32(i32 [[CONV]], i32 -32768)
+; CHECK-NEXT: [[SPEC_SELECT_I64:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP1]], i32 32767)
+; CHECK-NEXT: [[CONV1:%.*]] = trunc nsw i32 [[SPEC_SELECT_I64]] to i16
+; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_066]], i64 2
+; CHECK-NEXT: store i16 [[CONV1]], ptr [[PDST_ADDR_066]], align 2, !tbaa [[SHORT_TBAA9]]
+; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[PIN_068]], i64 16
+; CHECK-NEXT: [[TMP2:%.*]] = load double, ptr [[INCDEC_PTR]], align 8, !tbaa [[DOUBLE_TBAA7]]
+; CHECK-NEXT: [[MUL4:%.*]] = fmul double [[TMP2]], 3.276800e+04
+; CHECK-NEXT: [[CONV5:%.*]] = fptosi double [[MUL4]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.smax.i32(i32 [[CONV5]], i32 -32768)
+; CHECK-NEXT: [[SPEC_SELECT_I59:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP3]], i32 32767)
+; CHECK-NEXT: [[CONV7:%.*]] = trunc nsw i32 [[SPEC_SELECT_I59]] to i16
+; CHECK-NEXT: [[INCDEC_PTR8:%.*]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_066]], i64 4
+; CHECK-NEXT: store i16 [[CONV7]], ptr [[INCDEC_PTR2]], align 2, !tbaa [[SHORT_TBAA9]]
+; CHECK-NEXT: [[INCDEC_PTR9:%.*]] = getelementptr inbounds nuw i8, ptr [[PIN_068]], i64 24
+; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[INCDEC_PTR3]], align 8, !tbaa [[DOUBLE_TBAA7]]
+; CHECK-NEXT: [[MUL10:%.*]] = fmul double [[TMP4]], 3.276800e+04
+; CHECK-NEXT: [[CONV11:%.*]] = fptosi double [[MUL10]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = tail call i32 @llvm.smax.i32(i32 [[CONV11]], i32 -32768)
+; CHECK-NEXT: [[SPEC_SELECT_I54:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP5]], i32 32767)
+; CHECK-NEXT: [[CONV13:%.*]] = trunc nsw i32 [[SPEC_SELECT_I54]] to i16
+; CHECK-NEXT: [[INCDEC_PTR14:%.*]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_066]], i64 6
+; CHECK-NEXT: store i16 [[CONV13]], ptr [[INCDEC_PTR8]], align 2, !tbaa [[SHORT_TBAA9]]
+; CHECK-NEXT: [[INCDEC_PTR15]] = getelementptr inbounds nuw i8, ptr [[PIN_068]], i64 32
+; CHECK-NEXT: [[TMP6:%.*]] = load double, ptr [[INCDEC_PTR9]], align 8, !tbaa [[DOUBLE_TBAA7]]
+; CHECK-NEXT: [[MUL16:%.*]] = fmul double [[TMP6]], 3.276800e+04
+; CHECK-NEXT: [[CONV17:%.*]] = fptosi double [[MUL16]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = tail call i32 @llvm.smax.i32(i32 [[CONV17]], i32 -32768)
+; CHECK-NEXT: [[SPEC_SELECT_I49:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP7]], i32 32767)
+; CHECK-NEXT: [[CONV19:%.*]] = trunc nsw i32 [[SPEC_SELECT_I49]] to i16
+; CHECK-NEXT: [[INCDEC_PTR20]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_066]], i64 8
+; CHECK-NEXT: store i16 [[CONV19]], ptr [[INCDEC_PTR14]], align 2, !tbaa [[SHORT_TBAA9]]
+; CHECK-NEXT: [[DEC]] = add nsw i32 [[BLKCNT_067]], -1
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END_LOOPEXIT]], label %[[WHILE_BODY1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT: [[INCDEC_PTR15_LCSSA:%.*]] = phi ptr [ [[INCDEC_PTR15]], %[[WHILE_BODY1]] ], [ [[TMP37]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[INCDEC_PTR20_LCSSA:%.*]] = phi ptr [ [[INCDEC_PTR20]], %[[WHILE_BODY1]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[WHILE_END]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: [[PDST_ADDR_0_LCSSA:%.*]] = phi ptr [ [[PDST]], %[[ENTRY]] ], [ [[INCDEC_PTR20_LCSSA]], %[[WHILE_END_LOOPEXIT]] ]
+; CHECK-NEXT: [[PIN_0_LCSSA:%.*]] = phi ptr [ [[PSRC]], %[[ENTRY]] ], [ [[INCDEC_PTR15_LCSSA]], %[[WHILE_END_LOOPEXIT]] ]
+; CHECK-NEXT: [[REM:%.*]] = and i32 [[BLOCKSIZE]], 3
+; CHECK-NEXT: [[CMP22_NOT70:%.*]] = icmp eq i32 [[REM]], 0
+; CHECK-NEXT: br i1 [[CMP22_NOT70]], label %[[WHILE_END32:.*]], label %[[WHILE_BODY24_PREHEADER:.*]]
+; CHECK: [[WHILE_BODY24_PREHEADER]]:
+; CHECK-NEXT: br label %[[WHILE_BODY24:.*]]
+; CHECK: [[WHILE_BODY24]]:
+; CHECK-NEXT: [[PIN_173:%.*]] = phi ptr [ [[INCDEC_PTR25:%.*]], %[[WHILE_BODY24]] ], [ [[PIN_0_LCSSA]], %[[WHILE_BODY24_PREHEADER]] ]
+; CHECK-NEXT: [[BLKCNT_172:%.*]] = phi i32 [ [[DEC31:%.*]], %[[WHILE_BODY24]] ], [ [[REM]], %[[WHILE_BODY24_PREHEADER]] ]
+; CHECK-NEXT: [[PDST_ADDR_171:%.*]] = phi ptr [ [[INCDEC_PTR30:%.*]], %[[WHILE_BODY24]] ], [ [[PDST_ADDR_0_LCSSA]], %[[WHILE_BODY24_PREHEADER]] ]
+; CHECK-NEXT: [[INCDEC_PTR25]] = getelementptr inbounds nuw i8, ptr [[PIN_173]], i64 8
+; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[PIN_173]], align 8, !tbaa [[DOUBLE_TBAA7]]
+; CHECK-NEXT: [[MUL26:%.*]] = fmul double [[TMP8]], 3.276800e+04
+; CHECK-NEXT: [[CONV27:%.*]] = fptosi double [[MUL26]] to i32
+; CHECK-NEXT: [[TMP9:%.*]] = tail call i32 @llvm.smax.i32(i32 [[CONV27]], i32 -32768)
+; CHECK-NEXT: [[SPEC_SELECT_I:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP9]], i32 32767)
+; CHECK-NEXT: [[CONV29:%.*]] = trunc nsw i32 [[SPEC_SELECT_I]] to i16
+; CHECK-NEXT: [[INCDEC_PTR30]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_171]], i64 2
+; CHECK-NEXT: store i16 [[CONV29]], ptr [[PDST_ADDR_171]], align 2, !tbaa [[SHORT_TBAA9]]
+; CHECK-NEXT: [[DEC31]] = add nsw i32 [[BLKCNT_172]], -1
+; CHECK-NEXT: [[CMP22_NOT:%.*]] = icmp eq i32 [[DEC31]], 0
+; CHECK-NEXT: br i1 [[CMP22_NOT]], label %[[WHILE_END32_LOOPEXIT:.*]], label %[[WHILE_BODY24]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[WHILE_END32_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[WHILE_END32]]
+; CHECK: [[WHILE_END32]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %shr = lshr i32 %blockSize, 2
+ %cmp.not65 = icmp eq i32 %shr, 0
+ br i1 %cmp.not65, label %while.end, label %while.body.preheader
+
+while.body.preheader: ; preds = %entry
+ br label %while.body
+
+while.body: ; preds = %while.body.preheader, %while.body
+ %pIn.068 = phi ptr [ %incdec.ptr15, %while.body ], [ %pSrc, %while.body.preheader ]
+ %blkCnt.067 = phi i32 [ %dec, %while.body ], [ %shr, %while.body.preheader ]
+ %pDst.addr.066 = phi ptr [ %incdec.ptr20, %while.body ], [ %pDst, %while.body.preheader ]
+ %incdec.ptr = getelementptr inbounds nuw i8, ptr %pIn.068, i64 8
+ %0 = load double, ptr %pIn.068, align 8, !tbaa !7
+ %mul = fmul double %0, 3.276800e+04
+ %conv = fptosi double %mul to i32
+ %1 = tail call i32 @llvm.smax.i32(i32 %conv, i32 -32768)
+ %spec.select.i64 = tail call i32 @llvm.smin.i32(i32 %1, i32 32767)
+ %conv1 = trunc nsw i32 %spec.select.i64 to i16
+ %incdec.ptr2 = getelementptr inbounds nuw i8, ptr %pDst.addr.066, i64 2
+ store i16 %conv1, ptr %pDst.addr.066, align 2, !tbaa !9
+ %incdec.ptr3 = getelementptr inbounds nuw i8, ptr %pIn.068, i64 16
+ %2 = load double, ptr %incdec.ptr, align 8, !tbaa !7
+ %mul4 = fmul double %2, 3.276800e+04
+ %conv5 = fptosi double %mul4 to i32
+ %3 = tail call i32 @llvm.smax.i32(i32 %conv5, i32 -32768)
+ %spec.select.i59 = tail call i32 @llvm.smin.i32(i32 %3, i32 32767)
+ %conv7 = trunc nsw i32 %spec.select.i59 to i16
+ %incdec.ptr8 = getelementptr inbounds nuw i8, ptr %pDst.addr.066, i64 4
+ store i16 %conv7, ptr %incdec.ptr2, align 2, !tbaa !9
+ %incdec.ptr9 = getelementptr inbounds nuw i8, ptr %pIn.068, i64 24
+ %4 = load double, ptr %incdec.ptr3, align 8, !tbaa !7
+ %mul10 = fmul double %4, 3.276800e+04
+ %conv11 = fptosi double %mul10 to i32
+ %5 = tail call i32 @llvm.smax.i32(i32 %conv11, i32 -32768)
+ %spec.select.i54 = tail call i32 @llvm.smin.i32(i32 %5, i32 32767)
+ %conv13 = trunc nsw i32 %spec.select.i54 to i16
+ %incdec.ptr14 = getelementptr inbounds nuw i8, ptr %pDst.addr.066, i64 6
+ store i16 %conv13, ptr %incdec.ptr8, align 2, !tbaa !9
+ %incdec.ptr15 = getelementptr inbounds nuw i8, ptr %pIn.068, i64 32
+ %6 = load double, ptr %incdec.ptr9, align 8, !tbaa !7
+ %mul16 = fmul double %6, 3.276800e+04
+ %conv17 = fptosi double %mul16 to i32
+ %7 = tail call i32 @llvm.smax.i32(i32 %conv17, i32 -32768)
+ %spec.select.i49 = tail call i32 @llvm.smin.i32(i32 %7, i32 32767)
+ %conv19 = trunc nsw i32 %spec.select.i49 to i16
+ %incdec.ptr20 = getelementptr inbounds nuw i8, ptr %pDst.addr.066, i64 8
+ store i16 %conv19, ptr %incdec.ptr14, align 2, !tbaa !9
+ %dec = add nsw i32 %blkCnt.067, -1
+ %cmp.not = icmp eq i32 %dec, 0
+ br i1 %cmp.not, label %while.end.loopexit, label %while.body, !llvm.loop !11
+
+while.end.loopexit: ; preds = %while.body
+ %incdec.ptr15.lcssa = phi ptr [ %incdec.ptr15, %while.body ]
+ %incdec.ptr20.lcssa = phi ptr [ %incdec.ptr20, %while.body ]
+ br label %while.end
+
+while.end: ; preds = %while.end.loopexit, %entry
+ %pDst.addr.0.lcssa = phi ptr [ %pDst, %entry ], [ %incdec.ptr20.lcssa, %while.end.loopexit ]
+ %pIn.0.lcssa = phi ptr [ %pSrc, %entry ], [ %incdec.ptr15.lcssa, %while.end.loopexit ]
+ %rem = and i32 %blockSize, 3
+ %cmp22.not70 = icmp eq i32 %rem, 0
+ br i1 %cmp22.not70, label %while.end32, label %while.body24.preheader
+
+while.body24.preheader: ; preds = %while.end
+ br label %while.body24
+
+while.body24: ; preds = %while.body24.preheader, %while.body24
+ %pIn.173 = phi ptr [ %incdec.ptr25, %while.body24 ], [ %pIn.0.lcssa, %while.body24.preheader ]
+ %blkCnt.172 = phi i32 [ %dec31, %while.body24 ], [ %rem, %while.body24.preheader ]
+ %pDst.addr.171 = phi ptr [ %incdec.ptr30, %while.body24 ], [ %pDst.addr.0.lcssa, %while.body24.preheader ]
+ %incdec.ptr25 = getelementptr inbounds nuw i8, ptr %pIn.173, i64 8
+ %8 = load double, ptr %pIn.173, align 8, !tbaa !7
+ %mul26 = fmul double %8, 3.276800e+04
+ %conv27 = fptosi double %mul26 to i32
+ %9 = tail call i32 @llvm.smax.i32(i32 %conv27, i32 -32768)
+ %spec.select.i = tail call i32 @llvm.smin.i32(i32 %9, i32 32767)
+ %conv29 = trunc nsw i32 %spec.select.i to i16
+ %incdec.ptr30 = getelementptr inbounds nuw i8, ptr %pDst.addr.171, i64 2
+ store i16 %conv29, ptr %pDst.addr.171, align 2, !tbaa !9
+ %dec31 = add nsw i32 %blkCnt.172, -1
+ %cmp22.not = icmp eq i32 %dec31, 0
+ br i1 %cmp22.not, label %while.end32.loopexit, label %while.body24, !llvm.loop !13
+
+while.end32.loopexit: ; preds = %while.body24
+ br label %while.end32
+
+while.end32: ; preds = %while.end32.loopexit, %while.end
+ ret void
+}
+
+; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare i32 @llvm.smax.i32(i32, i32) #1
+
+; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare i32 @llvm.smin.i32(i32, i32) #1
+
+attributes #0 = { nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+!llvm.errno.tbaa = !{!3}
+
+!0 = !{i32 7, !"uwtable", i32 2}
+!1 = !{i32 7, !"frame-pointer", i32 4}
+!2 = !{!"clang version 23.0.0git (git at github.com:kamleshbhalui/llvm-project.git 448b725bb78bc288c5f6e3ad27856c92902f6537)"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"double", !5, i64 0}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"short", !5, i64 0}
+!11 = distinct !{!11, !12}
+!12 = !{!"llvm.loop.mustprogress"}
+!13 = distinct !{!13, !12}
+;.
+; CHECK: [[META5:![0-9]+]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
+; CHECK: [[META6]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[DOUBLE_TBAA7]] = !{[[META8:![0-9]+]], [[META8]], i64 0}
+; CHECK: [[META8]] = !{!"double", [[META5]], i64 0}
+; CHECK: [[SHORT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; CHECK: [[META10]] = !{!"short", [[META5]], i64 0}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META12:![0-9]+]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; CHECK: [[META12]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META12]], [[META14]], [[META13]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META12]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
index 7ce07a82e6465..916cba1609221 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*(INTERLEAVE-GROUP|REPLICATE|WIDEN)" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_2
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_4
; RUN: opt -passes=loop-vectorize -force-vector-width=8 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_8
@@ -9,15 +10,49 @@ target triple = "aarch64"
%i8.2 = type {i8, i8}
define void @i8_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i8_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i8_factor_2'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_8-LABEL: 'i8_factor_2'
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i8_factor_2'
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_8-LABEL: Checking a loop in 'i8_factor_2'
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i8_factor_2'
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
@@ -36,18 +71,41 @@ for.end:
%i16.2 = type {i16, i16}
define void @i16_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i16_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i16_factor_2'
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i16_factor_2'
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i16_factor_2'
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_4-LABEL: Checking a loop in 'i16_factor_2'
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i16_factor_2'
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i16_factor_2'
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
@@ -66,21 +124,33 @@ for.end:
%i32.2 = type {i32, i32}
define void @i32_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i32_factor_2'
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_4-LABEL: 'i32_factor_2'
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i32_factor_2'
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i32_factor_2'
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i32_factor_2'
-; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4-LABEL: Checking a loop in 'i32_factor_2'
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i32_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i32_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
@@ -99,21 +169,33 @@ for.end:
%i64.2 = type {i64, i64}
define void @i64_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i64_factor_2'
+; VF_2: Cost of 1 for VF 2: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 1 for VF 2: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_2: Cost of 1 for VF 2: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 1 for VF 2: WIDEN store ir<%tmp0>, ir<%tmp2>
+;
+; VF_4-LABEL: 'i64_factor_2'
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i64_factor_2'
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i64_factor_2'
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i64_factor_2'
-; VF_2: Cost of 1 for VF 2: WIDEN ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 1 for VF 2: WIDEN store ir<%tmp0>, ir<%tmp2>
-; VF_4-LABEL: Checking a loop in 'i64_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i64_factor_2'
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i64_factor_2'
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 0
@@ -132,6 +214,62 @@ for.end:
%i64.8 = type {i64, i64, i64, i64, i64, i64, i64, i64}
define void @i64_factor_8(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i64_factor_8'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 8 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i64_factor_8'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 16 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_8-LABEL: 'i64_factor_8'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_8: Cost of 32 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 32 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_8: Cost of 32 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_8: Cost of 32 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_8: Cost of 32 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 32 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_8: Cost of 32 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_8: Cost of 32 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_16-LABEL: 'i64_factor_8'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_16: Cost of 64 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 64 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_16: Cost of 64 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_16: Cost of 64 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<6>
+; VF_16: Cost of 64 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 64 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_16: Cost of 64 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_16: Cost of 64 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
entry:
br label %for.body
@@ -140,12 +278,6 @@ entry:
; implementation for determining the cost of the interleaved load group. The
; stores do not form a legal interleaved group because the group would contain
; gaps.
-;
-; VF_2-LABEL: Checking a loop in 'i64_factor_8'
-; VF_2: Cost of 8 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 8 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 8 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_2-NEXT: Cost of 8 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i64.8, ptr %data, i64 %i, i32 2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll b/llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll
index b5bf3def7fdce..4542beb782436 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -force-vector-interleave=1 -store-to-load-forwarding-conflict-detection=false -passes=loop-vectorize -S | FileCheck %s
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
@@ -28,11 +29,33 @@ target triple = "aarch64--linux-gnu"
; }
; }
-; CHECK-LABEL: @max_vf
-; CHECK: load <4 x i32>
-; CHECK: store <4 x i32>
define void @max_vf(ptr noalias nocapture %p) {
+; CHECK-LABEL: define void @max_vf(
+; CHECK-SAME: ptr noalias captures(none) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_PAIR:%.*]], ptr [[P]], i64 [[INDEX]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_PAIR]], ptr [[P]], i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[STRIDED_VEC]], <2 x i32> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: store <4 x i32> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_EXIT:.*]]
+; CHECK: [[FOR_EXIT]]:
+; CHECK-NEXT: ret void
+;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
index fdca980fcc773..3c7f19944449e 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*(INTERLEAVE-GROUP|REPLICATE|WIDEN)" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_2
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_4
; RUN: opt -passes=loop-vectorize -force-vector-width=8 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_8
@@ -9,15 +10,39 @@ target triple = "armv8--linux-gnueabihf"
%i8.2 = type {i8, i8}
define void @i8_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i8_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 25 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 14 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 14 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 25 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 14 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 14 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i8_factor_2'
+; VF_4: Cost of 49 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 49 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 49 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 49 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i8_factor_2'
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i8_factor_2'
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_8-LABEL: Checking a loop in 'i8_factor_2'
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i8_factor_2'
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
@@ -36,18 +61,33 @@ for.end:
%i16.2 = type {i16, i16}
define void @i16_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i16_factor_2'
+; VF_2: Cost of 25 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 25 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 25 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 25 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_4-LABEL: 'i16_factor_2'
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i16_factor_2'
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i16_factor_2'
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_4-LABEL: Checking a loop in 'i16_factor_2'
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i16_factor_2'
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i16_factor_2'
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
@@ -66,21 +106,33 @@ for.end:
%i32.2 = type {i32, i32}
define void @i32_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i32_factor_2'
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_4-LABEL: 'i32_factor_2'
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i32_factor_2'
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i32_factor_2'
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i32_factor_2'
-; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4-LABEL: Checking a loop in 'i32_factor_2'
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i32_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i32_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
@@ -99,15 +151,33 @@ for.end:
%half.2 = type {half, half}
define void @half_factor_2(ptr %data, i64 %n) {
+; VF_2-LABEL: 'half_factor_2'
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_4-LABEL: 'half_factor_2'
+; VF_4: Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'half_factor_2'
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'half_factor_2'
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_4-LABEL: Checking a loop in 'half_factor_2'
-; VF_4: Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'half_factor_2'
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %half.2, ptr %data, i64 %i, i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
index 7f5114fc4aa27..4c19c36f232bb 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*(INTERLEAVE-GROUP|REPLICATE|WIDEN)" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_2
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_4
; RUN: opt -passes=loop-vectorize -force-vector-width=8 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_8
@@ -11,23 +12,41 @@ target triple = "thumbv8.1m.main-none-eabi"
%i8.2 = type {i8, i8}
define void @i8_factor_2(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i8_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i8_factor_2'
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i8_factor_2'
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i8_factor_2'
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i8_factor_2'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_4-LABEL: Checking a loop in 'i8_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i8_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i8_factor_2'
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
@@ -46,23 +65,41 @@ for.end:
%i16.2 = type {i16, i16}
define void @i16_factor_2(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i16_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i16_factor_2'
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i16_factor_2'
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i16_factor_2'
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i16_factor_2'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_4-LABEL: Checking a loop in 'i16_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i16_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i16_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i16.2, ptr %data, i64 %i, i32 0
@@ -81,23 +118,41 @@ for.end:
%i32.2 = type {i32, i32}
define void @i32_factor_2(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i32_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i32_factor_2'
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'i32_factor_2'
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'i32_factor_2'
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i32_factor_2'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_4-LABEL: Checking a loop in 'i32_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'i32_factor_2'
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'i32_factor_2'
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i32.2, ptr %data, i64 %i, i32 0
@@ -116,29 +171,65 @@ for.end:
%i64.2 = type {i64, i64}
define void @i64_factor_2(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i64_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'i64_factor_2'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_8-LABEL: 'i64_factor_2'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_16-LABEL: 'i64_factor_2'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i64_factor_2'
-; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 22 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_4-LABEL: Checking a loop in 'i64_factor_2'
-; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 44 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_8-LABEL: Checking a loop in 'i64_factor_2'
-; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 88 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_16-LABEL: Checking a loop in 'i64_factor_2'
-; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 176 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i64.2, ptr %data, i64 %i, i32 0
@@ -157,23 +248,41 @@ for.end:
%f16.2 = type {half, half}
define void @f16_factor_2(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f16_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'f16_factor_2'
+; VF_4: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'f16_factor_2'
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'f16_factor_2'
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f16_factor_2'
-; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_4-LABEL: Checking a loop in 'f16_factor_2'
-; VF_4: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'f16_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'f16_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f16.2, ptr %data, i64 %i, i32 0
@@ -192,21 +301,33 @@ for.end:
%f32.2 = type {float, float}
define void @f32_factor_2(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f32_factor_2'
+; VF_2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_4-LABEL: 'f32_factor_2'
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_8-LABEL: 'f32_factor_2'
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
+; VF_16-LABEL: 'f32_factor_2'
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f32_factor_2'
-; VF_2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4-LABEL: Checking a loop in 'f32_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'f32_factor_2'
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'f32_factor_2'
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f32.2, ptr %data, i64 %i, i32 0
@@ -225,29 +346,65 @@ for.end:
%f64.2 = type {double, double}
define void @f64_factor_2(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f64_factor_2'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_4-LABEL: 'f64_factor_2'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_8-LABEL: 'f64_factor_2'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
+; VF_16-LABEL: 'f64_factor_2'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f64_factor_2'
-; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_4-LABEL: Checking a loop in 'f64_factor_2'
-; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_8-LABEL: Checking a loop in 'f64_factor_2'
-; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp1>
-; VF_16-LABEL: Checking a loop in 'f64_factor_2'
-; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp2> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp2>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp1>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f64.2, ptr %data, i64 %i, i32 0
@@ -270,37 +427,89 @@ for.end:
%i8.3 = type {i8, i8, i8}
define void @i8_factor_3(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i8_factor_3'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_4-LABEL: 'i8_factor_3'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_8-LABEL: 'i8_factor_3'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_16-LABEL: 'i8_factor_3'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i8_factor_3'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_4-LABEL: Checking a loop in 'i8_factor_3'
-; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_8-LABEL: Checking a loop in 'i8_factor_3'
-; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_16-LABEL: Checking a loop in 'i8_factor_3'
-; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
@@ -322,37 +531,89 @@ for.end:
%i16.3 = type {i16, i16, i16}
define void @i16_factor_3(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i16_factor_3'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_4-LABEL: 'i16_factor_3'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_8-LABEL: 'i16_factor_3'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_16-LABEL: 'i16_factor_3'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i16_factor_3'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_4-LABEL: Checking a loop in 'i16_factor_3'
-; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_8-LABEL: Checking a loop in 'i16_factor_3'
-; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_16-LABEL: Checking a loop in 'i16_factor_3'
-; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 0
@@ -374,37 +635,91 @@ for.end:
%i32.3 = type {i32, i32, i32}
define void @i32_factor_3(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i32_factor_3'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_4-LABEL: 'i32_factor_3'
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp5>
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp5>
+;
+; VF_8-LABEL: 'i32_factor_3'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_16-LABEL: 'i32_factor_3'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i32_factor_3'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_4-LABEL: Checking a loop in 'i32_factor_3'
-; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp3> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp3>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp4>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp5>
-; VF_8-LABEL: Checking a loop in 'i32_factor_3'
-; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_16-LABEL: Checking a loop in 'i32_factor_3'
-; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 0
@@ -426,37 +741,89 @@ for.end:
%i64.3 = type {i64, i64, i64}
define void @i64_factor_3(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i64_factor_3'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_4-LABEL: 'i64_factor_3'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_8-LABEL: 'i64_factor_3'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_16-LABEL: 'i64_factor_3'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i64_factor_3'
-; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 22 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 22 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_4-LABEL: Checking a loop in 'i64_factor_3'
-; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 44 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 44 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_8-LABEL: Checking a loop in 'i64_factor_3'
-; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 88 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 88 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_16-LABEL: Checking a loop in 'i64_factor_3'
-; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 176 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 176 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 0
@@ -478,25 +845,47 @@ for.end:
%f16.3 = type {half, half, half}
define void @f16_factor_3(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f16_factor_3'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_4-LABEL: 'f16_factor_3'
+; VF_4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_8-LABEL: 'f16_factor_3'
+; VF_8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'f16_factor_3'
+; VF_16: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f16_factor_3'
-; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_4-LABEL: Checking a loop in 'f16_factor_3'
-; VF_4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'f16_factor_3'
-; VF_8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'f16_factor_3'
-; VF_16: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_16: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f16.3, ptr %data, i64 %i, i32 0
@@ -518,25 +907,49 @@ for.end:
%f32.3 = type {float, float, float}
define void @f32_factor_3(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f32_factor_3'
+; VF_2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_4-LABEL: 'f32_factor_3'
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp5>
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp5>
+;
+; VF_8-LABEL: 'f32_factor_3'
+; VF_8: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'f32_factor_3'
+; VF_16: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f32_factor_3'
-; VF_2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_4-LABEL: Checking a loop in 'f32_factor_3'
-; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp3> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp3>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp4>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp5>
-; VF_8-LABEL: Checking a loop in 'f32_factor_3'
-; VF_8: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_8: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'f32_factor_3'
-; VF_16: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
-; VF_16: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f32.3, ptr %data, i64 %i, i32 0
@@ -558,37 +971,89 @@ for.end:
%f64.3 = type {double, double, double}
define void @f64_factor_3(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f64_factor_3'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_4-LABEL: 'f64_factor_3'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_8-LABEL: 'f64_factor_3'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
+; VF_16-LABEL: 'f64_factor_3'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f64_factor_3'
-; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_4-LABEL: Checking a loop in 'f64_factor_3'
-; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_8-LABEL: Checking a loop in 'f64_factor_3'
-; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp2>
-; VF_16-LABEL: Checking a loop in 'f64_factor_3'
-; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp3> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp3>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp2>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f64.3, ptr %data, i64 %i, i32 0
@@ -613,45 +1078,113 @@ for.end:
%i8.4 = type {i8, i8, i8, i8}
define void @i8_factor_4(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i8_factor_4'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_4-LABEL: 'i8_factor_4'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_8-LABEL: 'i8_factor_4'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_16-LABEL: 'i8_factor_4'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i8_factor_4'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_4-LABEL: Checking a loop in 'i8_factor_4'
-; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_8-LABEL: Checking a loop in 'i8_factor_4'
-; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_16-LABEL: Checking a loop in 'i8_factor_4'
-; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 0
@@ -676,45 +1209,113 @@ for.end:
%i16.4 = type {i16, i16, i16, i16}
define void @i16_factor_4(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i16_factor_4'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_4-LABEL: 'i16_factor_4'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_8-LABEL: 'i16_factor_4'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_16-LABEL: 'i16_factor_4'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i16_factor_4'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_4-LABEL: Checking a loop in 'i16_factor_4'
-; VF_4: Cost of 24 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 24 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_8-LABEL: Checking a loop in 'i16_factor_4'
-; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_16-LABEL: Checking a loop in 'i16_factor_4'
-; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i16.4, ptr %data, i64 %i, i32 0
@@ -739,45 +1340,115 @@ for.end:
%i32.4 = type {i32, i32, i32, i32}
define void @i32_factor_4(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i32_factor_4'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_4-LABEL: 'i32_factor_4'
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp3> = getelementptr inbounds ir<%data>, ir<%i>, ir<3>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp5>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp6>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp3>, ir<%tmp7>
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp3> = getelementptr inbounds ir<%data>, ir<%i>, ir<3>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp5>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp6>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp3>, ir<%tmp7>
+;
+; VF_8-LABEL: 'i32_factor_4'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_16-LABEL: 'i32_factor_4'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i32_factor_4'
-; VF_2: Cost of 12 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 12 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_4-LABEL: Checking a loop in 'i32_factor_4'
-; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp6> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp7> = load ir<%tmp3>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp4>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp5>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp6>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp3>, ir<%tmp7>
-; VF_8-LABEL: Checking a loop in 'i32_factor_4'
-; VF_8: Cost of 48 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 48 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_16-LABEL: Checking a loop in 'i32_factor_4'
-; VF_16: Cost of 96 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 96 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i32.4, ptr %data, i64 %i, i32 0
@@ -802,45 +1473,113 @@ for.end:
%i64.4 = type {i64, i64, i64, i64}
define void @i64_factor_4(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'i64_factor_4'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 6 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_4-LABEL: 'i64_factor_4'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 12 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_8-LABEL: 'i64_factor_4'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 24 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_16-LABEL: 'i64_factor_4'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 48 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'i64_factor_4'
-; VF_2: Cost of 22 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 22 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 22 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 22 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_4-LABEL: Checking a loop in 'i64_factor_4'
-; VF_4: Cost of 44 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 44 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 44 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 44 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_8-LABEL: Checking a loop in 'i64_factor_4'
-; VF_8: Cost of 88 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 88 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 88 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 88 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_16-LABEL: Checking a loop in 'i64_factor_4'
-; VF_16: Cost of 176 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 176 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 176 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 176 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %i64.4, ptr %data, i64 %i, i32 0
@@ -865,21 +1604,33 @@ for.end:
%f16.4 = type {half, half, half, half}
define void @f16_factor_4(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f16_factor_4'
+; VF_2: Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_2: Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_2: Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_2: Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+;
+; VF_4-LABEL: 'f16_factor_4'
+; VF_4: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_4: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_4: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_4: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+;
+; VF_8-LABEL: 'f16_factor_4'
+; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+;
+; VF_16-LABEL: 'f16_factor_4'
+; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f16_factor_4'
-; VF_2: Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_2: Cost of 18 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_4-LABEL: Checking a loop in 'f16_factor_4'
-; VF_4: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_4: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8-LABEL: Checking a loop in 'f16_factor_4'
-; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'f16_factor_4'
-; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f16.4, ptr %data, i64 %i, i32 0
@@ -904,27 +1655,55 @@ for.end:
%f32.4 = type {float, float, float, float}
define void @f32_factor_4(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f32_factor_4'
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+;
+; VF_4-LABEL: 'f32_factor_4'
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp3> = getelementptr inbounds ir<%data>, ir<%i>, ir<3>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp5>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp6>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp3>, ir<%tmp7>
+; VF_4: Cost of 0 for VF 4: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp0> = getelementptr inbounds ir<%data>, ir<%i>, ir<0>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp1> = getelementptr inbounds ir<%data>, ir<%i>, ir<1>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp2> = getelementptr inbounds ir<%data>, ir<%i>, ir<2>
+; VF_4: Cost of 0 for VF 4: WIDEN-GEP ir<%tmp3> = getelementptr inbounds ir<%data>, ir<%i>, ir<3>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp5>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp6>
+; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp3>, ir<%tmp7>
+;
+; VF_8-LABEL: 'f32_factor_4'
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+;
+; VF_16-LABEL: 'f32_factor_4'
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f32_factor_4'
-; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_2: Cost of 20 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_4-LABEL: Checking a loop in 'f32_factor_4'
-; VF_4: Cost of 8 for VF 4: WIDEN ir<%tmp4> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp5> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp6> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN ir<%tmp7> = load ir<%tmp3>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp4>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp1>, ir<%tmp5>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp2>, ir<%tmp6>
-; VF_4-NEXT: Cost of 8 for VF 4: WIDEN store ir<%tmp3>, ir<%tmp7>
-; VF_8-LABEL: Checking a loop in 'f32_factor_4'
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16-LABEL: Checking a loop in 'f32_factor_4'
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f32.4, ptr %data, i64 %i, i32 0
@@ -949,45 +1728,113 @@ for.end:
%f64.4 = type {double, double, double, double}
define void @f64_factor_4(ptr %data, i64 %n) #0 {
+; VF_2-LABEL: 'f64_factor_4'
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_2: Cost of 0 for VF 2: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_4-LABEL: 'f64_factor_4'
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_4: Cost of 0 for VF 4: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_4: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_8-LABEL: 'f64_factor_4'
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_8: Cost of 0 for VF 8: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_8: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
+; VF_16-LABEL: 'f64_factor_4'
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5:%[0-9]+]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp0> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<0>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp1> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<1>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp2> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<2>
+; VF_16: Cost of 0 for VF 16: REPLICATE ir<%tmp3> = getelementptr inbounds ir<%data>, vp<[[VP5]]>, ir<3>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
+; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
+; VF_16: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
+;
entry:
br label %for.body
-; VF_2-LABEL: Checking a loop in 'f64_factor_4'
-; VF_2: Cost of 6 for VF 2: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_2-NEXT: Cost of 6 for VF 2: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_2-NEXT: Cost of 4 for VF 2: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_4-LABEL: Checking a loop in 'f64_factor_4'
-; VF_4: Cost of 12 for VF 4: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_4-NEXT: Cost of 12 for VF 4: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_4-NEXT: Cost of 8 for VF 4: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_8-LABEL: Checking a loop in 'f64_factor_4'
-; VF_8: Cost of 24 for VF 8: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_8-NEXT: Cost of 24 for VF 8: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_8-NEXT: Cost of 16 for VF 8: REPLICATE store ir<%tmp7>, ir<%tmp3>
-; VF_16-LABEL: Checking a loop in 'f64_factor_4'
-; VF_16: Cost of 48 for VF 16: REPLICATE ir<%tmp4> = load ir<%tmp0>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE ir<%tmp5> = load ir<%tmp1>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE ir<%tmp6> = load ir<%tmp2>
-; VF_16-NEXT: Cost of 48 for VF 16: REPLICATE ir<%tmp7> = load ir<%tmp3>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp4>, ir<%tmp0>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp5>, ir<%tmp1>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp6>, ir<%tmp2>
-; VF_16-NEXT: Cost of 32 for VF 16: REPLICATE store ir<%tmp7>, ir<%tmp3>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds %f64.4, ptr %data, i64 %i, i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
index f86af8777a03c..1b0a14ef9e556 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*INTERLEAVE-GROUP" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,-optimized-nf2-segment-load-store -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NO-OPT
; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=OPT-NF2
@@ -18,52 +19,200 @@
%i8.2 = type {i8, i8}
define void @i8_factor_2(ptr %data, i64 %n) {
+; NO-OPT-LABEL: 'i8_factor_2'
+; NO-OPT: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 16 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 16 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 32 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 32 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 64 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 64 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; OPT-NF2-LABEL: 'i8_factor_2'
+; OPT-NF2: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; OPT-NF3-LABEL: 'i8_factor_2'
+; OPT-NF3: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; OPT-NF4-LABEL: 'i8_factor_2'
+; OPT-NF4: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; OPT-NF5-LABEL: 'i8_factor_2'
+; OPT-NF5: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; OPT-NF6-LABEL: 'i8_factor_2'
+; OPT-NF6: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; OPT-NF7-LABEL: 'i8_factor_2'
+; OPT-NF7: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; OPT-NF8-LABEL: 'i8_factor_2'
+; OPT-NF8: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>, vp<%evl>
+;
+; FIXED-NO-OPT-LABEL: 'i8_factor_2'
+; FIXED-NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
+; FIXED-OPT-NF2-LABEL: 'i8_factor_2'
+; FIXED-OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
+; FIXED-OPT-NF3-LABEL: 'i8_factor_2'
+; FIXED-OPT-NF3: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF3: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
+; FIXED-OPT-NF4-LABEL: 'i8_factor_2'
+; FIXED-OPT-NF4: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF4: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
+; FIXED-OPT-NF5-LABEL: 'i8_factor_2'
+; FIXED-OPT-NF5: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF5: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
+; FIXED-OPT-NF6-LABEL: 'i8_factor_2'
+; FIXED-OPT-NF6: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF6: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
+; FIXED-OPT-NF7-LABEL: 'i8_factor_2'
+; FIXED-OPT-NF7: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF7: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
+; FIXED-OPT-NF8-LABEL: 'i8_factor_2'
+; FIXED-OPT-NF8: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+; FIXED-OPT-NF8: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
+;
entry:
br label %for.body
-; FIXED-OPT-NF2-LABEL: Checking a loop in 'i8_factor_2'
-; FIXED-OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2-LABEL: Checking a loop in 'i8_factor_2'
-; OPT-NF2: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 3 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; OPT-NF2: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_2'
-; FIXED-NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT-LABEL: Checking a loop in 'i8_factor_2'
-; NO-OPT: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 16 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 16 for VF vscale x 4: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 32 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 32 for VF vscale x 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 64 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; NO-OPT: Cost of 64 for VF vscale x 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%p0 = getelementptr inbounds %i8.2, ptr %data, i64 %i, i32 0
@@ -84,52 +233,200 @@ for.end:
%i8.3 = type {i8, i8, i8}
define void @i8_factor_3(ptr %data, i64 %n) {
+; NO-OPT-LABEL: 'i8_factor_3'
+; NO-OPT: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; OPT-NF2-LABEL: 'i8_factor_3'
+; OPT-NF2: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; OPT-NF3-LABEL: 'i8_factor_3'
+; OPT-NF3: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 4 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 4 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 5 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 5 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 7 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 7 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 14 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 14 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; OPT-NF4-LABEL: 'i8_factor_3'
+; OPT-NF4: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; OPT-NF5-LABEL: 'i8_factor_3'
+; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; OPT-NF6-LABEL: 'i8_factor_3'
+; OPT-NF6: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; OPT-NF7-LABEL: 'i8_factor_3'
+; OPT-NF7: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; OPT-NF8-LABEL: 'i8_factor_3'
+; OPT-NF8: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>, vp<%evl>
+;
+; FIXED-NO-OPT-LABEL: 'i8_factor_3'
+; FIXED-NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
+; FIXED-OPT-NF2-LABEL: 'i8_factor_3'
+; FIXED-OPT-NF2: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF2: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
+; FIXED-OPT-NF3-LABEL: 'i8_factor_3'
+; FIXED-OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
+; FIXED-OPT-NF4-LABEL: 'i8_factor_3'
+; FIXED-OPT-NF4: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF4: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
+; FIXED-OPT-NF5-LABEL: 'i8_factor_3'
+; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF5: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
+; FIXED-OPT-NF6-LABEL: 'i8_factor_3'
+; FIXED-OPT-NF6: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF6: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
+; FIXED-OPT-NF7-LABEL: 'i8_factor_3'
+; FIXED-OPT-NF7: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF7: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
+; FIXED-OPT-NF8-LABEL: 'i8_factor_3'
+; FIXED-OPT-NF8: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+; FIXED-OPT-NF8: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
+;
entry:
br label %for.body
-; FIXED-OPT-NF3-LABEL: Checking a loop in 'i8_factor_3'
-; FIXED-OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3-LABEL: Checking a loop in 'i8_factor_3'
-; OPT-NF3: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 4 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 4 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 5 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 5 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 7 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 7 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 14 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; OPT-NF3: Cost of 14 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_3'
-; FIXED-NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; FIXED-NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT-LABEL: Checking a loop in 'i8_factor_3'
-; NO-OPT: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
-; NO-OPT: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3, ir<%p0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%p0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
@@ -154,52 +451,200 @@ for.end:
%i8.4 = type {i8, i8, i8, i8}
define void @i8_factor_4(ptr %data, i64 %n) {
+; NO-OPT-LABEL: 'i8_factor_4'
+; NO-OPT: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; OPT-NF2-LABEL: 'i8_factor_4'
+; OPT-NF2: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; OPT-NF3-LABEL: 'i8_factor_4'
+; OPT-NF3: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; OPT-NF4-LABEL: 'i8_factor_4'
+; OPT-NF4: Cost of 5 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 5 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 5 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 5 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 6 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 6 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 8 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 8 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 16 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 16 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; OPT-NF5-LABEL: 'i8_factor_4'
+; OPT-NF5: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; OPT-NF6-LABEL: 'i8_factor_4'
+; OPT-NF6: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; OPT-NF7-LABEL: 'i8_factor_4'
+; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; OPT-NF8-LABEL: 'i8_factor_4'
+; OPT-NF8: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>, vp<%evl>
+;
+; FIXED-NO-OPT-LABEL: 'i8_factor_4'
+; FIXED-NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
+; FIXED-OPT-NF2-LABEL: 'i8_factor_4'
+; FIXED-OPT-NF2: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF2: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
+; FIXED-OPT-NF3-LABEL: 'i8_factor_4'
+; FIXED-OPT-NF3: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF3: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
+; FIXED-OPT-NF4-LABEL: 'i8_factor_4'
+; FIXED-OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
+; FIXED-OPT-NF5-LABEL: 'i8_factor_4'
+; FIXED-OPT-NF5: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF5: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
+; FIXED-OPT-NF6-LABEL: 'i8_factor_4'
+; FIXED-OPT-NF6: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF6: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
+; FIXED-OPT-NF7-LABEL: 'i8_factor_4'
+; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF7: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
+; FIXED-OPT-NF8-LABEL: 'i8_factor_4'
+; FIXED-OPT-NF8: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+; FIXED-OPT-NF8: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
+;
entry:
br label %for.body
-; FIXED-OPT-NF4-LABEL: Checking a loop in 'i8_factor_4'
-; FIXED-OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4-LABEL: Checking a loop in 'i8_factor_4'
-; OPT-NF4: Cost of 5 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 5 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 5 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 5 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 6 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 6 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 8 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 8 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 16 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; OPT-NF4: Cost of 16 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_4'
-; FIXED-NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT-LABEL: Checking a loop in 'i8_factor_4'
-; NO-OPT: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; NO-OPT: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%p0 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 0
@@ -228,44 +673,168 @@ for.end:
%i8.5 = type {i8, i8, i8, i8, i8}
define void @i8_factor_5(ptr %data, i64 %n) {
+; NO-OPT-LABEL: 'i8_factor_5'
+; NO-OPT: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; OPT-NF2-LABEL: 'i8_factor_5'
+; OPT-NF2: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; OPT-NF3-LABEL: 'i8_factor_5'
+; OPT-NF3: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; OPT-NF4-LABEL: 'i8_factor_5'
+; OPT-NF4: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; OPT-NF5-LABEL: 'i8_factor_5'
+; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 7 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 7 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 9 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 9 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 13 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 13 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; OPT-NF6-LABEL: 'i8_factor_5'
+; OPT-NF6: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; OPT-NF7-LABEL: 'i8_factor_5'
+; OPT-NF7: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; OPT-NF8-LABEL: 'i8_factor_5'
+; OPT-NF8: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>, vp<%evl>
+;
+; FIXED-NO-OPT-LABEL: 'i8_factor_5'
+; FIXED-NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
+; FIXED-OPT-NF2-LABEL: 'i8_factor_5'
+; FIXED-OPT-NF2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF2: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF2: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF2: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF2: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF2: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF2: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
+; FIXED-OPT-NF3-LABEL: 'i8_factor_5'
+; FIXED-OPT-NF3: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF3: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF3: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF3: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF3: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF3: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF3: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF3: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
+; FIXED-OPT-NF4-LABEL: 'i8_factor_5'
+; FIXED-OPT-NF4: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF4: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF4: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF4: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF4: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF4: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF4: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF4: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
+; FIXED-OPT-NF5-LABEL: 'i8_factor_5'
+; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
+; FIXED-OPT-NF6-LABEL: 'i8_factor_5'
+; FIXED-OPT-NF6: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF6: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF6: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF6: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF6: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF6: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF6: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF6: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
+; FIXED-OPT-NF7-LABEL: 'i8_factor_5'
+; FIXED-OPT-NF7: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF7: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF7: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF7: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF7: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF7: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF7: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF7: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
+; FIXED-OPT-NF8-LABEL: 'i8_factor_5'
+; FIXED-OPT-NF8: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF8: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF8: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF8: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF8: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF8: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF8: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+; FIXED-OPT-NF8: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
+;
entry:
br label %for.body
-; FIXED-OPT-NF5-LABEL: Checking a loop in 'i8_factor_5'
-; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5-LABEL: Checking a loop in 'i8_factor_5'
-; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5: Cost of 7 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5: Cost of 7 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5: Cost of 9 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5: Cost of 9 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5: Cost of 13 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; OPT-NF5: Cost of 13 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_5'
-; FIXED-NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; FIXED-NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT-LABEL: Checking a loop in 'i8_factor_5'
-; NO-OPT: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
-; NO-OPT: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5, ir<%p0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%p0 = getelementptr inbounds %i8.5, ptr %data, i64 %i, i32 0
@@ -298,44 +867,168 @@ for.end:
%i8.6 = type {i8, i8, i8, i8, i8, i8}
define void @i8_factor_6(ptr %data, i64 %n) {
+; NO-OPT-LABEL: 'i8_factor_6'
+; NO-OPT: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; OPT-NF2-LABEL: 'i8_factor_6'
+; OPT-NF2: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; OPT-NF3-LABEL: 'i8_factor_6'
+; OPT-NF3: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; OPT-NF4-LABEL: 'i8_factor_6'
+; OPT-NF4: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; OPT-NF5-LABEL: 'i8_factor_6'
+; OPT-NF5: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; OPT-NF6-LABEL: 'i8_factor_6'
+; OPT-NF6: Cost of 7 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 7 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 10 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 10 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 14 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 14 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; OPT-NF7-LABEL: 'i8_factor_6'
+; OPT-NF7: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; OPT-NF8-LABEL: 'i8_factor_6'
+; OPT-NF8: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>, vp<%evl>
+;
+; FIXED-NO-OPT-LABEL: 'i8_factor_6'
+; FIXED-NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
+; FIXED-OPT-NF2-LABEL: 'i8_factor_6'
+; FIXED-OPT-NF2: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF2: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF2: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF2: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF2: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF2: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF2: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF2: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
+; FIXED-OPT-NF3-LABEL: 'i8_factor_6'
+; FIXED-OPT-NF3: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF3: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF3: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF3: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF3: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF3: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF3: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF3: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
+; FIXED-OPT-NF4-LABEL: 'i8_factor_6'
+; FIXED-OPT-NF4: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF4: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF4: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF4: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF4: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF4: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF4: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF4: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
+; FIXED-OPT-NF5-LABEL: 'i8_factor_6'
+; FIXED-OPT-NF5: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF5: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF5: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF5: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF5: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF5: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF5: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF5: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
+; FIXED-OPT-NF6-LABEL: 'i8_factor_6'
+; FIXED-OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
+; FIXED-OPT-NF7-LABEL: 'i8_factor_6'
+; FIXED-OPT-NF7: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF7: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF7: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF7: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF7: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF7: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF7: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF7: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
+; FIXED-OPT-NF8-LABEL: 'i8_factor_6'
+; FIXED-OPT-NF8: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF8: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF8: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF8: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF8: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF8: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF8: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+; FIXED-OPT-NF8: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
+;
entry:
br label %for.body
-; FIXED-OPT-NF6-LABEL: Checking a loop in 'i8_factor_6'
-; FIXED-OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6-LABEL: Checking a loop in 'i8_factor_6'
-; OPT-NF6: Cost of 7 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6: Cost of 7 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6: Cost of 10 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6: Cost of 10 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6: Cost of 14 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; OPT-NF6: Cost of 14 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_6'
-; FIXED-NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; FIXED-NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT-LABEL: Checking a loop in 'i8_factor_6'
-; NO-OPT: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
-; NO-OPT: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6, ir<%p0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%p0 = getelementptr inbounds %i8.6, ptr %data, i64 %i, i32 0
@@ -372,44 +1065,168 @@ for.end:
%i8.7 = type {i8, i8, i8, i8, i8, i8, i8}
define void @i8_factor_7(ptr %data, i64 %n) {
+; NO-OPT-LABEL: 'i8_factor_7'
+; NO-OPT: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; OPT-NF2-LABEL: 'i8_factor_7'
+; OPT-NF2: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; OPT-NF3-LABEL: 'i8_factor_7'
+; OPT-NF3: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; OPT-NF4-LABEL: 'i8_factor_7'
+; OPT-NF4: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; OPT-NF5-LABEL: 'i8_factor_7'
+; OPT-NF5: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; OPT-NF6-LABEL: 'i8_factor_7'
+; OPT-NF6: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; OPT-NF7-LABEL: 'i8_factor_7'
+; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 9 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 9 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 11 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 11 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 15 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 15 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; OPT-NF8-LABEL: 'i8_factor_7'
+; OPT-NF8: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>, vp<%evl>
+;
+; FIXED-NO-OPT-LABEL: 'i8_factor_7'
+; FIXED-NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
+; FIXED-OPT-NF2-LABEL: 'i8_factor_7'
+; FIXED-OPT-NF2: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF2: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF2: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF2: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF2: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF2: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF2: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF2: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
+; FIXED-OPT-NF3-LABEL: 'i8_factor_7'
+; FIXED-OPT-NF3: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF3: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF3: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF3: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF3: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF3: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF3: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF3: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
+; FIXED-OPT-NF4-LABEL: 'i8_factor_7'
+; FIXED-OPT-NF4: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF4: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF4: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF4: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF4: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF4: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF4: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
+; FIXED-OPT-NF5-LABEL: 'i8_factor_7'
+; FIXED-OPT-NF5: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF5: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF5: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF5: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF5: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF5: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF5: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF5: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
+; FIXED-OPT-NF6-LABEL: 'i8_factor_7'
+; FIXED-OPT-NF6: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF6: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF6: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF6: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF6: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF6: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF6: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF6: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
+; FIXED-OPT-NF7-LABEL: 'i8_factor_7'
+; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
+; FIXED-OPT-NF8-LABEL: 'i8_factor_7'
+; FIXED-OPT-NF8: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF8: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF8: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF8: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF8: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF8: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+; FIXED-OPT-NF8: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
+;
entry:
br label %for.body
-; FIXED-OPT-NF7-LABEL: Checking a loop in 'i8_factor_7'
-; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7-LABEL: Checking a loop in 'i8_factor_7'
-; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7: Cost of 9 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7: Cost of 9 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7: Cost of 11 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7: Cost of 11 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7: Cost of 15 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; OPT-NF7: Cost of 15 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_7'
-; FIXED-NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; FIXED-NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT-LABEL: Checking a loop in 'i8_factor_7'
-; NO-OPT: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
-; NO-OPT: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7, ir<%p0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%p0 = getelementptr inbounds %i8.7, ptr %data, i64 %i, i32 0
@@ -450,44 +1267,168 @@ for.end:
%i8.8 = type {i8, i8, i8, i8, i8, i8, i8, i8}
define void @i8_factor_8(ptr %data, i64 %n) {
+; NO-OPT-LABEL: 'i8_factor_8'
+; NO-OPT: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; NO-OPT: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; OPT-NF2-LABEL: 'i8_factor_8'
+; OPT-NF2: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF2: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; OPT-NF3-LABEL: 'i8_factor_8'
+; OPT-NF3: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF3: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; OPT-NF4-LABEL: 'i8_factor_8'
+; OPT-NF4: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF4: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; OPT-NF5-LABEL: 'i8_factor_8'
+; OPT-NF5: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF5: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; OPT-NF6-LABEL: 'i8_factor_8'
+; OPT-NF6: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF6: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; OPT-NF7-LABEL: 'i8_factor_8'
+; OPT-NF7: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF7: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; OPT-NF8-LABEL: 'i8_factor_8'
+; OPT-NF8: Cost of 9 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 9 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 10 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 10 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 12 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 12 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 16 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+; OPT-NF8: Cost of 16 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>, vp<%evl>
+;
+; FIXED-NO-OPT-LABEL: 'i8_factor_8'
+; FIXED-NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
+; FIXED-OPT-NF2-LABEL: 'i8_factor_8'
+; FIXED-OPT-NF2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF2: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF2: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF2: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF2: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF2: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF2: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF2: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
+; FIXED-OPT-NF3-LABEL: 'i8_factor_8'
+; FIXED-OPT-NF3: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF3: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF3: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF3: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF3: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF3: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF3: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF3: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
+; FIXED-OPT-NF4-LABEL: 'i8_factor_8'
+; FIXED-OPT-NF4: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF4: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF4: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF4: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF4: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF4: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF4: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF4: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
+; FIXED-OPT-NF5-LABEL: 'i8_factor_8'
+; FIXED-OPT-NF5: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF5: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF5: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF5: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF5: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF5: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF5: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF5: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
+; FIXED-OPT-NF6-LABEL: 'i8_factor_8'
+; FIXED-OPT-NF6: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF6: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF6: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF6: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF6: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF6: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF6: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF6: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
+; FIXED-OPT-NF7-LABEL: 'i8_factor_8'
+; FIXED-OPT-NF7: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF7: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF7: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF7: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF7: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF7: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF7: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF7: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
+; FIXED-OPT-NF8-LABEL: 'i8_factor_8'
+; FIXED-OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+; FIXED-OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
+;
entry:
br label %for.body
-; FIXED-OPT-NF8-LABEL: Checking a loop in 'i8_factor_8'
-; FIXED-OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8-LABEL: Checking a loop in 'i8_factor_8'
-; OPT-NF8: Cost of 9 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8: Cost of 9 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8: Cost of 10 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8: Cost of 10 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8: Cost of 12 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8: Cost of 12 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8: Cost of 16 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; OPT-NF8: Cost of 16 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_8'
-; FIXED-NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; FIXED-NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT-LABEL: Checking a loop in 'i8_factor_8'
-; NO-OPT: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
-; NO-OPT: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8, ir<%p0>
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%p0 = getelementptr inbounds %i8.8, ptr %data, i64 %i, i32 0
>From dbfef9494025a6645834d95e9847f2d9c4f1501e Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 25 Jun 2026 17:12:14 +0000
Subject: [PATCH 2/2] [LoopVectorize] Extend narrow interleave groups
optimization to intrinsics
Narrowing full interleave groups is valid when the interleave factor matches
the selected VF, it does not require the group width to match the target vector
register width. Removing that constraint lets the cost model consider the
narrowed plan for more VFs.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 91 ++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 38 +-
.../CodeGen/WebAssembly/memory-interleave.ll | 445 +++++-------------
.../AArch64/foldshuffleidentity.ll | 39 +-
.../AArch64/interleave-with-runtime-checks.ll | 8 +-
.../LoopVectorize/AArch64/interleaved_cost.ll | 72 +--
...-narrow-interleave-to-widen-memory-cost.ll | 32 +-
...-interleave-to-widen-memory-derived-ivs.ll | 55 +--
...interleave-to-widen-memory-epilogue-vec.ll | 186 +++++++-
...eave-to-widen-memory-remove-loop-region.ll | 33 +-
...terleave-to-widen-memory-with-live-outs.ll | 6 +-
...to-widen-memory-with-wide-ops-and-casts.ll | 40 +-
...nterleave-to-widen-memory-with-wide-ops.ll | 81 ++--
...sform-narrow-interleave-to-widen-memory.ll | 211 +++++++--
.../LoopVectorize/ARM/interleaved_cost.ll | 64 +--
.../LoopVectorize/ARM/mve-interleaved-cost.ll | 120 ++---
.../RISCV/interleaved-accesses.ll | 26 +-
.../LoopVectorize/RISCV/interleaved-cost.ll | 32 --
...sform-narrow-interleave-to-widen-memory.ll | 66 +--
.../WebAssembly/memory-interleave.ll | 180 +------
.../X86/CostModel/interleaved-load-half.ll | 4 -
...interleave-to-widen-memory-epilogue-vec.ll | 40 +-
...sform-narrow-interleave-to-widen-memory.ll | 162 ++++---
.../interleaved-accesses-metadata.ll | 13 +-
llvm/test/Transforms/LoopVectorize/pr31098.ll | 2 +-
.../AArch64/interleavevectorization.ll | 43 +-
26 files changed, 1029 insertions(+), 1060 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index adcfe30ff9561..1c6cc9d3d0168 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5779,13 +5779,14 @@ static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
return false;
}
-static bool canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable) {
+static bool canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable,
+ const TargetTransformInfo &TTI) {
SmallVector<VPValue *> Ops0;
auto *WideMember0 = dyn_cast<VPRecipeWithIRFlags>(Ops[0]);
if (!WideMember0)
return false;
for (VPValue *V : Ops) {
- if (!isa<VPWidenRecipe, VPWidenCastRecipe>(V))
+ if (!isa<VPWidenRecipe, VPWidenCastRecipe, VPWidenIntrinsicRecipe>(V))
return false;
auto *R = cast<VPRecipeWithIRFlags>(V);
if (getOpcodeOrIntrinsicID(R) != getOpcodeOrIntrinsicID(WideMember0))
@@ -5801,7 +5802,16 @@ static bool canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable) {
for (VPValue *Op : Ops)
OpsI.push_back(Op->getDefiningRecipe()->getOperand(Idx));
- if (canNarrowOps(OpsI, IsScalable))
+ auto *WideIntrinsic0 = dyn_cast<VPWidenIntrinsicRecipe>(WideMember0);
+ if (WideIntrinsic0 &&
+ isVectorIntrinsicWithScalarOpAtArg(
+ WideIntrinsic0->getVectorIntrinsicID(), Idx, &TTI)) {
+ if (!all_of(OpsI, equal_to(OpsI.front())))
+ return false;
+ continue;
+ }
+
+ if (canNarrowOps(OpsI, IsScalable, TTI))
continue;
if (any_of(enumerate(OpsI), [WideMember0, Idx, IsScalable](const auto &P) {
@@ -5854,8 +5864,10 @@ isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
for (ElementCount VF : VFs) {
unsigned MinVal = VF.getKnownMinValue();
+
unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal;
- if (IG->getFactor() == MinVal && GroupSize == GetVectorBitWidthForVF(VF))
+ if (MinVal % IG->getFactor() == 0 &&
+ GroupSize % GetVectorBitWidthForVF(VF) == 0)
return {VF};
}
return std::nullopt;
@@ -5872,10 +5884,14 @@ static bool isAlreadyNarrow(VPValue *VPV) {
// Convert the wide recipes defining the VPValues in \p Members feeding an
// interleave group to a single narrow variant. The first member is reused as
// the narrowed recipe. BuildVectors for live-in operands are inserted into \p
-// Preheader.
+// Preheader. \p RepeatFactor is the number of original iterations the narrowed
+// plan processes at once (VF / interleave factor); per-field live-ins are
+// repeated this many times so the assembled BuildVector spans all VF lanes.
static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
SmallPtrSetImpl<VPValue *> &NarrowedOps,
- VPBasicBlock *Preheader) {
+ VPBasicBlock *Preheader,
+ const TargetTransformInfo &TTI,
+ unsigned RepeatFactor) {
VPValue *V = Members.front();
auto *R = V->getDefiningRecipe();
if (NarrowedOps.contains(V))
@@ -5888,7 +5904,11 @@ static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
M->getScalarType() == V->getScalarType();
}) &&
"expected distinct live-ins of matching scalar type");
- auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
+ SmallVector<VPValue *> Vals;
+ Vals.reserve(Members.size() * RepeatFactor);
+ for (unsigned I = 0; I != RepeatFactor; ++I)
+ Vals.append(Members.begin(), Members.end());
+ auto *BV = new VPInstruction(VPInstruction::BuildVector, Vals);
Preheader->appendRecipe(BV);
NarrowedOps.insert(BV);
return BV;
@@ -5897,7 +5917,7 @@ static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
if (isAlreadyNarrow(V))
return V;
- if (isa<VPWidenRecipe, VPWidenCastRecipe>(R)) {
+ if (isa<VPWidenRecipe, VPWidenCastRecipe, VPWidenIntrinsicRecipe>(R)) {
auto *WideMember0 = cast<VPRecipeWithIRFlags>(R);
for (VPValue *Member : Members.drop_front())
WideMember0->intersectFlags(*cast<VPRecipeWithIRFlags>(Member));
@@ -5905,8 +5925,18 @@ static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
SmallVector<VPValue *> OpsI;
for (VPValue *Member : Members)
OpsI.push_back(Member->getDefiningRecipe()->getOperand(Idx));
- WideMember0->setOperand(
- Idx, narrowInterleaveGroupOp(OpsI, NarrowedOps, Preheader));
+ auto *WideIntrinsic0 = dyn_cast<VPWidenIntrinsicRecipe>(WideMember0);
+ if (WideIntrinsic0 &&
+ isVectorIntrinsicWithScalarOpAtArg(
+ WideIntrinsic0->getVectorIntrinsicID(), Idx, &TTI)) {
+ assert(all_of(OpsI, equal_to(OpsI.front())) &&
+ "scalar intrinsic operands must match");
+ WideMember0->setOperand(Idx, OpsI.front());
+ continue;
+ }
+ WideMember0->setOperand(Idx, narrowInterleaveGroupOp(OpsI, NarrowedOps,
+ Preheader, TTI,
+ RepeatFactor));
}
return V;
}
@@ -5998,7 +6028,9 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
// when checking all allowed consumers (store interleave groups) below.
if (!InterleaveR)
continue;
-
+ // Skip read interleave groups.
+ if (InterleaveR->getStoredValues().empty())
+ continue;
// Try to find a single VF, where all interleave groups are consecutive and
// saturate the full vector width. If we already have a candidate VF, check
// if it is applicable for the current InterleaveR, otherwise look for a
@@ -6012,10 +6044,6 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
return nullptr;
VFToOptimize = NarrowedVF;
- // Skip read interleave groups.
- if (InterleaveR->getStoredValues().empty())
- continue;
-
// Narrow interleave groups, if all operands are already matching narrow
// ops.
auto *Member0 = InterleaveR->getStoredValues()[0];
@@ -6042,7 +6070,7 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
// Check if all values feeding InterleaveR are matching wide recipes, which
// operands that can be narrowed.
if (!canNarrowOps(InterleaveR->getStoredValues(),
- VFToOptimize->isScalable()))
+ VFToOptimize->isScalable(), TTI))
return nullptr;
StoreGroups.push_back(InterleaveR);
}
@@ -6050,6 +6078,12 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
if (StoreGroups.empty())
return nullptr;
+ unsigned CommonInterleaveFactor =
+ StoreGroups.front()->getInterleaveGroup()->getFactor();
+ if (any_of(StoreGroups, [CommonInterleaveFactor](auto *R) {
+ return R->getInterleaveGroup()->getFactor() != CommonInterleaveFactor;
+ }))
+ return nullptr;
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
bool RequiresScalarEpilogue =
MiddleVPBB->getNumSuccessors() == 1 &&
@@ -6072,10 +6106,16 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
// Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
SmallPtrSet<VPValue *, 4> NarrowedOps;
VPBasicBlock *Preheader = Plan.getVectorPreheader();
+ // The narrowed plan processes VF / factor original iterations per vector
+ // iteration; per-field live-ins are repeated this many times when assembled
+ // into BuildVectors.
+ unsigned RepeatFactor =
+ VFToOptimize->getKnownMinValue() / CommonInterleaveFactor;
// Narrow operation tree rooted at store groups.
for (auto *StoreGroup : StoreGroups) {
- VPValue *Res = narrowInterleaveGroupOp(StoreGroup->getStoredValues(),
- NarrowedOps, Preheader);
+ VPValue *Res =
+ narrowInterleaveGroupOp(StoreGroup->getStoredValues(), NarrowedOps,
+ Preheader, TTI, RepeatFactor);
auto *SI =
cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos());
auto *S = new VPWidenStoreRecipe(*SI, StoreGroup->getAddr(), Res, nullptr,
@@ -6094,16 +6134,11 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
VPValue *UF = &Plan.getUF();
VPValue *Step;
- if (VFToOptimize->isScalable()) {
- VPValue *VScale =
- PHBuilder.createElementCount(CanIVTy, ElementCount::getScalable(1));
- Step = PHBuilder.createOverflowingOp(Instruction::Mul, {VScale, UF},
- {true, false});
- Plan.getVF().replaceAllUsesWith(VScale);
- } else {
- Step = UF;
- Plan.getVF().replaceAllUsesWith(Plan.getConstantInt(CanIVTy, 1));
- }
+ VPValue *ScaleStep = PHBuilder.createElementCount(
+ CanIVTy, ElementCount::get(RepeatFactor, VFToOptimize->isScalable()));
+ Step = PHBuilder.createOverflowingOp(Instruction::Mul, {ScaleStep, UF},
+ {true, false});
+ Plan.getVF().replaceAllUsesWith(ScaleStep);
// Materialize vector trip count with the narrowed step.
materializeVectorTripCount(Plan, VectorPH, /*TailByMasking=*/false,
RequiresScalarEpilogue, Step);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 7e0f536a63a83..410c9d63f4435 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -684,33 +684,37 @@ VPInstruction *vputils::findCanonicalIVIncrement(VPlan &Plan) {
if (!VFxUF.isMaterialized())
return Step == &VFxUF;
+ // After narrowing interleave groups, the step is
+ // ScaleStep * UF, with ScaleStep = VF / InterleaveFactor.
VPSymbolicValue &UF = Plan.getUF();
if (!UF.isMaterialized())
- return Step == &UF ||
- match(Step, m_c_Mul(m_Specific(&Plan.getUF()),
- m_VPInstruction<VPInstruction::VScale>()));
+ return Step == &UF || match(Step, m_c_Mul(m_Specific(&UF), m_VPValue()));
// Alias masking: step is number of active lanes of a dependence mask.
if (match(Step, m_ZExtOrTruncOrSelf(
m_VPInstruction<VPInstruction::NumActiveLanes>())))
return true;
+ // UF is now concrete, so the step has folded to ScaleStep * ConcreteUF.
+ // Extract its constant coefficient, treating fixed and scalable VF
+ // uniformly by ignoring any vscale factor (present only for scalable VF).
unsigned ConcreteUF = Plan.getConcreteUF();
- // Fixed VF: step is just the concrete UF.
- if (match(Step, m_SpecificInt(ConcreteUF)))
- return true;
+ uint64_t Coeff = 0;
+ if (match(Step, m_Shl(m_VPInstruction<VPInstruction::VScale>(),
+ m_ConstantInt(Coeff))))
+ // mul(vscale, Coeff) may have been simplified to shl(vscale, log2(Coeff))
+ // when Coeff is a power of 2.
+ Coeff = uint64_t(1) << Coeff;
+ else if (match(Step, m_VPInstruction<VPInstruction::VScale>()))
+ // Bare vscale: coefficient is 1.
+ Coeff = 1;
+ else if (!match(Step, m_ConstantInt(Coeff)) &&
+ !match(Step, m_c_Mul(m_ConstantInt(Coeff),
+ m_VPInstruction<VPInstruction::VScale>())))
+ return false;
- // Scalable VF: step involves VScale.
- if (ConcreteUF == 1)
- return match(Step, m_VPInstruction<VPInstruction::VScale>());
- if (match(Step, m_c_Mul(m_SpecificInt(ConcreteUF),
- m_VPInstruction<VPInstruction::VScale>())))
- return true;
- // mul(VScale, ConcreteUF) may have been simplified to
- // shl(VScale, log2(ConcreteUF)) when ConcreteUF is a power of 2.
- return isPowerOf2_32(ConcreteUF) &&
- match(Step, m_Shl(m_VPInstruction<VPInstruction::VScale>(),
- m_SpecificInt(Log2_32(ConcreteUF))));
+ // The step is ScaleStep * ConcreteUF.
+ return Coeff != 0 && Coeff % ConcreteUF == 0;
};
VPInstruction *Increment = nullptr;
diff --git a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
index 2b7d53fa31d2b..281d98d702cfb 100644
--- a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
+++ b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
@@ -22,28 +22,18 @@ target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20
define hidden void @two_ints_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: two_ints_same_op:
; CHECK: loop # label2:
-; CHECK: v128.load 0:p2align=2
-; CHECK: v128.load 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load 0:p2align=2
-; CHECK: v128.load 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store 0:p2align=2
-; CHECK: end_loop
-; CHECK: loop # label3:
; CHECK: i32.load 0
; CHECK: i32.load 0
; CHECK: i32.store 0
; CHECK: i32.load 0
; CHECK: i32.load 0
; CHECK: i32.store 0
+; CHECK: end_loop
+; CHECK: loop # label3:
+; CHECK: v128.load64_zero 0:p2align=2
+; CHECK: v128.load64_zero 0:p2align=2
+; CHECK: i32x4.add
+; CHECK: v128.store64_lane 0:p2align=2, 0
; CHECK: end_loop
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -128,7 +118,6 @@ define hidden void @two_ints_vary_op(ptr noalias nocapture noundef writeonly %0,
define hidden void @three_ints(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: three_ints:
; CHECK: loop # label9:
-; CHECK-NOT: v128.load
; CHECK: i32.load 0
; CHECK: i32.load 0
; CHECK: i32.store 0
@@ -176,7 +165,6 @@ define hidden void @three_ints(ptr noalias nocapture noundef writeonly %0, ptr n
define hidden void @three_shorts(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: three_shorts:
; CHECK: loop # label11:
-; CHECK-NOT: v128.load
; CHECK: i32.load16_u 0
; CHECK: i32.load16_u 0
; CHECK: i32.store16 0
@@ -224,30 +212,10 @@ define hidden void @three_shorts(ptr noalias nocapture noundef writeonly %0, ptr
define hidden void @four_shorts_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: four_shorts_same_op:
; CHECK: loop # label14:
-; CHECK: v128.load 0:p2align=1
-; CHECK: v128.load 16:p2align=1
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: v128.load 0:p2align=1
-; CHECK: v128.load 16:p2align=1
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
+; CHECK: v128.load64_zero 0:p2align=1
+; CHECK: v128.load64_zero 0:p2align=1
; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i16x8.sub
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 4, 5, 20, 21, 0, 1, 0, 1, 6, 7, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store 16:p2align=1
-; CHECK: i8x16.shuffle 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19, 0, 1, 0, 1
-; CHECK: i8x16.shuffle 0, 1, 0, 1, 0, 1, 16, 17, 0, 1, 0, 1, 2, 3, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31
-; CHECK: v128.store 0:p2align=1
+; CHECK: v128.store64_lane 0:p2align=1, 0
; CHECK: end_loop
; CHECK: loop # label15:
; CHECK: i32.load16_u 0
@@ -473,7 +441,6 @@ define hidden void @four_shorts_interleave_op(ptr noalias nocapture noundef writ
define hidden void @five_shorts(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: five_shorts:
; CHECK: loop # label25:
-; CHECK-NOT: v128.load
; CHECK: i32.load16_u 0:p2align=0
; CHECK: i32.load16_u 0:p2align=0
; CHECK: i32.store16 0:p2align=0
@@ -744,30 +711,9 @@ define hidden void @three_bytes_interleave_op(ptr noalias nocapture noundef writ
define hidden void @four_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: four_bytes_same_op:
; CHECK: loop # label40:
-; CHECK: v128.load 0:p2align=0
-; CHECK: v128.load 16:p2align=0
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.load 0:p2align=0
-; CHECK: v128.load 16:p2align=0
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: v128.and
-; CHECK: i8x16.shuffle 0, 0, 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23
-; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
-; CHECK: v128.store 16:p2align=0
-; CHECK: i8x16.shuffle 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19, 0, 0
-; CHECK: i8x16.shuffle 0, 0, 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19
-; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
-; CHECK: v128.store 0:p2align=0
+; CHECK: i32.load 0:p2align=0
+; CHECK: i32.load 0:p2align=0
+; CHECK: i32.store 0:p2align=0
; CHECK: end_loop
; CHECK: loop # label41:
; CHECK: i32.load8_u 0
@@ -990,8 +936,14 @@ define hidden void @four_bytes_interleave_op(ptr noalias nocapture noundef write
define hidden void @eight_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: eight_bytes_same_op:
-; CHECK: loop # label51:
-; CHECK-NOT: v128.load
+; CHECK: loop # label52:
+; CHECK: v128.load64_zero 0:p2align=0
+; CHECK: v128.load64_zero 0:p2align=0
+; CHECK: i16x8.extmul_low_i8x16_u
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.store64_lane 0:p2align=0, 0
+; CHECK: end_loop
+; CHECK: loop # label53:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.store8 0
@@ -1088,8 +1040,7 @@ define hidden void @eight_bytes_same_op(ptr noalias nocapture noundef writeonly
define hidden void @eight_bytes_split_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: eight_bytes_split_op:
-; CHECK: loop # label53:
-; CHECK-NOT: v128.load
+; CHECK: loop # label55:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.store8 0
@@ -1186,8 +1137,7 @@ define hidden void @eight_bytes_split_op(ptr noalias nocapture noundef writeonly
define hidden void @eight_bytes_interleave_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: eight_bytes_interleave_op:
-; CHECK: loop # label55:
-; CHECK-NOT: v128.load
+; CHECK: loop # label57:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.store8 0
@@ -1284,67 +1234,17 @@ define hidden void @eight_bytes_interleave_op(ptr noalias nocapture noundef writ
define hidden void @four_bytes_into_four_ints_same_op(ptr noalias nocapture noundef %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: four_bytes_into_four_ints_same_op:
-; CHECK: loop # label58:
-; CHECK: v128.load 0:p2align=0
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: loop # label60:
+; CHECK: v128.load32_zero 0:p2align=0
; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: v128.load 0:p2align=0
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load32_zero 0:p2align=0
; CHECK: i16x8.extend_low_i8x16_u
; CHECK: i32x4.extmul_low_i16x8_u
; CHECK: v128.load 0:p2align=2
-; CHECK: v128.load 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: v128.load 32:p2align=2
-; CHECK: v128.load 48:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extmul_low_i16x8_u
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extmul_low_i16x8_u
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_u
-; CHECK: i32x4.extmul_low_i16x8_u
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 32:p2align=2
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 48:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
; CHECK: v128.store 0:p2align=2
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 16:p2align=2
; CHECK: end_loop
-; CHECK: loop # label59:
+; CHECK: loop # label61:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.load 0
@@ -1421,7 +1321,7 @@ define hidden void @four_bytes_into_four_ints_same_op(ptr noalias nocapture noun
define hidden void @four_bytes_into_four_ints_vary_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
; CHECK-LABEL: four_bytes_into_four_ints_vary_op:
-; CHECK: loop # label62:
+; CHECK: loop # label64:
; CHECK: v128.load 0:p2align=0
; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_u
@@ -1464,7 +1364,7 @@ define hidden void @four_bytes_into_four_ints_vary_op(ptr noalias nocapture noun
; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label63:
+; CHECK: loop # label65:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.store 0
@@ -1528,13 +1428,13 @@ define hidden void @four_bytes_into_four_ints_vary_op(ptr noalias nocapture noun
define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
; CHECK-LABEL: scale_uv_row_down2:
-; CHECK: loop # label67:
+; CHECK: loop # label69:
; CHECK: v128.load 0:p2align=0
; CHECK: v128.load 16:p2align=0
; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29
; CHECK: v128.store 0:p2align=0
; CHECK: end_loop
-; CHECK: loop # label68:
+; CHECK: loop # label70:
; CHECK: i32.load8_u 0
; CHECK: i32.store8 0
; CHECK: i32.load8_u 0
@@ -1566,7 +1466,7 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
; CHECK-LABEL: scale_uv_row_down2_box:
-; CHECK: loop # label72:
+; CHECK: loop # label74:
; CHECK: v128.load 0:p2align=0
; CHECK: v128.load 16:p2align=0
; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
@@ -1601,7 +1501,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
; CHECK: i8x16.shuffle 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30
; CHECK: v128.store 0:p2align=0
; CHECK: end_loop
-; CHECK: loop # label73:
+; CHECK: loop # label75:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
@@ -1676,7 +1576,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
; CHECK-LABEL: scale_uv_row_down2_linear:
-; CHECK: loop # label78:
+; CHECK: loop # label80:
; CHECK: v128.load 0:p2align=0
; CHECK: v128.load 16:p2align=0
; CHECK: i8x16.shuffle 0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0
@@ -1688,7 +1588,7 @@ define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0,
; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
; CHECK: v128.store 0:p2align=0
; CHECK: end_loop
-; CHECK: loop # label79:
+; CHECK: loop # label81:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.store8 0
@@ -1737,7 +1637,7 @@ define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0,
define hidden void @scale_argb_row_down2_box(ptr noundef readonly captures(none) %arg, i32 noundef %arg1, ptr noundef writeonly captures(none) %arg2, i32 noundef %arg3) local_unnamed_addr #0 {
; CHECK-LABEL: scale_argb_row_down2_box:
-; CHECK: loop # label83:
+; CHECK: loop # label85:
; CHECK: v128.load 0:p2align=0
; CHECK: v128.load 16:p2align=0
; CHECK: i8x16.shuffle 0, 8, 16, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
@@ -1868,7 +1768,7 @@ define hidden void @scale_argb_row_down2_box(ptr noundef readonly captures(none)
; CHECK: i8x16.replace_lane 15
; CHECK: v128.store 0:p2align=0
; CHECK: end_loop
-; CHECK: loop # label84:
+; CHECK: loop # label86:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
@@ -2006,7 +1906,7 @@ define hidden void @argb_to_uv_matrix_row(ptr noundef readonly captures(none) %a
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: v128.load8_splat 33
; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: loop # label88:
+; CHECK: loop # label90:
; CHECK: v128.load 0:p2align=0
; CHECK: v128.load 16:p2align=0
; CHECK: i8x16.shuffle 1, 9, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
@@ -2123,7 +2023,7 @@ define hidden void @argb_to_uv_matrix_row(ptr noundef readonly captures(none) %a
; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: v128.store64_lane 0:p2align=0, 0
; CHECK: end_loop
-; CHECK: loop # label89:
+; CHECK: loop # label91:
; CHECK: i32.load16_u 128
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
@@ -2297,23 +2197,13 @@ bb134: ; preds = %bb18, %bb
define hidden void @two_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_floats_same_op:
-; CHECK: loop # label92:
-; CHECK: v128.load 0:p2align=2
-; CHECK: v128.load 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load 0:p2align=2
-; CHECK: v128.load 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: loop # label94:
+; CHECK: v128.load64_zero 0:p2align=2
+; CHECK: v128.load64_zero 0:p2align=2
; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store 0:p2align=2
+; CHECK: v128.store64_lane 0:p2align=2, 0
; CHECK: end_loop
-; CHECK: loop # label93:
+; CHECK: loop # label95:
; CHECK: end_loop
entry:
%cmp21.not = icmp eq i32 %N, 0
@@ -2345,7 +2235,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_floats_vary_op:
-; CHECK: loop # label96:
+; CHECK: loop # label98:
; CHECK: v128.load 0:p2align=2
; CHECK: v128.load 16:p2align=2
; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
@@ -2361,7 +2251,7 @@ define hidden void @two_floats_vary_op(ptr noundef readonly captures(none) %a, p
; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label97:
+; CHECK: loop # label99:
; CHECK: end_loop
entry:
%cmp20.not = icmp eq i32 %N, 0
@@ -2393,7 +2283,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_bytes_two_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_bytes_two_floats_same_op:
-; CHECK: loop # label100:
+; CHECK: loop # label102:
; CHECK: v128.load64_zero 0:p2align=0
; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
@@ -2419,7 +2309,7 @@ define hidden void @two_bytes_two_floats_same_op(ptr noundef readonly captures(n
; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label101:
+; CHECK: loop # label103:
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
@@ -2459,7 +2349,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_bytes_two_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_bytes_two_floats_vary_op:
-; CHECK: loop # label104:
+; CHECK: loop # label106:
; CHECK: v128.load64_zero 0:p2align=0
; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
@@ -2485,7 +2375,7 @@ define hidden void @two_bytes_two_floats_vary_op(ptr noundef readonly captures(n
; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label105:
+; CHECK: loop # label107:
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
@@ -2526,7 +2416,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_floats_two_bytes_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_floats_two_bytes_same_op:
; CHECK: i16x8.narrow_i32x4_u
-; CHECK: loop # label108:
+; CHECK: loop # label110:
; CHECK: v128.load 0:p2align=2
; CHECK: v128.load 16:p2align=2
; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
@@ -2549,7 +2439,7 @@ define hidden void @two_floats_two_bytes_same_op(ptr noundef readonly captures(n
; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: v128.store64_lane 0:p2align=0, 0
; CHECK: end_loop
-; CHECK: loop # label109:
+; CHECK: loop # label111:
; CHECK: i32.store8 0
; CHECK: i32.store8 0
; CHECK: end_loop
@@ -2586,7 +2476,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_floats_two_bytes_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_floats_two_bytes_vary_op:
; CHECK: i16x8.narrow_i32x4_u
-; CHECK: loop # label112:
+; CHECK: loop # label114:
; CHECK: v128.load 0:p2align=2
; CHECK: v128.load 16:p2align=2
; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
@@ -2609,7 +2499,7 @@ define hidden void @two_floats_two_bytes_vary_op(ptr noundef readonly captures(n
; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: v128.store64_lane 0:p2align=0, 0
; CHECK: end_loop
-; CHECK: loop # label113:
+; CHECK: loop # label115:
; CHECK: i32.store8 0
; CHECK: i32.store8 0
; CHECK: end_loop
@@ -2645,29 +2535,21 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_shorts_two_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_shorts_two_floats_same_op:
-; CHECK: loop # label116:
-; CHECK: v128.load 0:p2align=1
-; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load 0:p2align=1
-; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
+; CHECK: loop # label118:
+; CHECK: v128.load32_zero 0:p2align=1
+; CHECK: i16x8.extract_lane_s 0
+; CHECK: f32x4.splat
+; CHECK: i16x8.extract_lane_s 1
+; CHECK: f32x4.replace_lane 1
+; CHECK: v128.load32_zero 0:p2align=1
+; CHECK: i16x8.extract_lane_s 0
+; CHECK: f32x4.splat
+; CHECK: i16x8.extract_lane_s 1
+; CHECK: f32x4.replace_lane 1
; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: v128.store 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: v128.store 0:p2align=2
+; CHECK: v128.store64_lane 0:p2align=2, 0
; CHECK: end_loop
-; CHECK: loop # label117:
+; CHECK: loop # label119:
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
@@ -2707,7 +2589,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_shorts_two_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_shorts_two_floats_vary_op:
-; CHECK: loop # label120:
+; CHECK: loop # label122:
; CHECK: v128.load 0:p2align=1
; CHECK: i8x16.shuffle 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 0, 1, 0, 1, 0, 1
; CHECK: i32x4.extend_low_i16x8_s
@@ -2729,7 +2611,7 @@ define hidden void @two_shorts_two_floats_vary_op(ptr noundef readonly captures(
; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label121:
+; CHECK: loop # label123:
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
@@ -2769,28 +2651,17 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_floats_two_shorts_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_floats_two_shorts_same_op:
-; CHECK: loop # label124:
-; CHECK: v128.load 0:p2align=2
-; CHECK: v128.load 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
-; CHECK: v128.load 0:p2align=2
-; CHECK: v128.load 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK: loop # label126:
+; CHECK: v128.load64_zero 0:p2align=2
+; CHECK: v128.load64_zero 0:p2align=2
; CHECK: f32x4.mul
; CHECK: i32x4.trunc_sat_f32x4_s
; CHECK: v128.const 65535, 65535, 65535, 65535
; CHECK: v128.and
; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-; CHECK: f32x4.mul
-; CHECK: i32x4.trunc_sat_f32x4_s
-; CHECK: v128.and
-; CHECK: i16x8.narrow_i32x4_u
-; CHECK: i8x16.shuffle 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
-; CHECK: v128.store 0:p2align=1
+; CHECK: v128.store32_lane 0:p2align=1, 0
; CHECK: end_loop
-; CHECK: loop # label125:
+; CHECK: loop # label127:
; CHECK: i32.store16 0
; CHECK: i32.store16 0
; CHECK: end_loop
@@ -2826,7 +2697,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @two_floats_two_shorts_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: two_floats_two_shorts_vary_op:
-; CHECK: loop # label128:
+; CHECK: loop # label130:
; CHECK: v128.load 0:p2align=2
; CHECK: v128.load 16:p2align=2
; CHECK: i8x16.shuffle 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
@@ -2847,7 +2718,7 @@ define hidden void @two_floats_two_shorts_vary_op(ptr noundef readonly captures(
; CHECK: i8x16.shuffle 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
; CHECK: v128.store 0:p2align=1
; CHECK: end_loop
-; CHECK: loop # label129:
+; CHECK: loop # label131:
; CHECK: i32.store16 0
; CHECK: i32.store16 0
; CHECK: end_loop
@@ -2883,13 +2754,13 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_same_op:
-; CHECK: loop # label132:
+; CHECK: loop # label134:
; CHECK: v128.load 0:p2align=2
; CHECK: v128.load 0:p2align=2
; CHECK: f32x4.mul
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label133:
+; CHECK: loop # label135:
; CHECK: end_loop
entry:
%cmp45.not = icmp eq i32 %N, 0
@@ -2935,8 +2806,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_vary_op:
-; CHECK-NOT: f32x4
-; CHECK: loop # label135:
+; CHECK: loop # label137:
; CHECK: end_loop
entry:
%cmp42.not = icmp eq i32 %N, 0
@@ -2982,63 +2852,19 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_bytes_four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_bytes_four_floats_same_op:
-; CHECK: loop # label138:
-; CHECK: v128.load 0:p2align=0
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: loop # label140:
+; CHECK: v128.load32_zero 0:p2align=0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load 0:p2align=0
-; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 1, 5, 9, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: v128.load32_zero 0:p2align=0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 6, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 3, 7, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: i16x8.extend_low_i8x16_s
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 48:p2align=2
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 32:p2align=2
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label139:
+; CHECK: loop # label141:
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
@@ -3100,7 +2926,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_bytes_four_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_bytes_four_floats_vary_op:
-; CHECK: loop # label142:
+; CHECK: loop # label144:
; CHECK: v128.load 0:p2align=0
; CHECK: i8x16.shuffle 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
@@ -3156,7 +2982,7 @@ define hidden void @four_bytes_four_floats_vary_op(ptr noundef readonly captures
; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label143:
+; CHECK: loop # label145:
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
; CHECK: i32.load8_s 0
@@ -3218,8 +3044,19 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_floats_four_bytes_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_bytes_same_op:
-; CHECK-NOT: v128.load
-; CHECK: loop # label145:
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: loop # label148:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 0:p2align=2
+; CHECK: f32x4.mul
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.const 255, 255, 255, 255
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: i8x16.narrow_i16x8_u
+; CHECK: v128.store32_lane 0:p2align=0, 0
+; CHECK: end_loop
+; CHECK: loop # label149:
; CHECK: i32.store8 0
; CHECK: i32.store8 0
; CHECK: i32.store8 0
@@ -3273,8 +3110,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_floats_four_bytes_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_bytes_vary_op:
-; CHECK-NOT: v128.load
-; CHECK: loop # label147:
+; CHECK: loop # label151:
; CHECK: i32.store8 0
; CHECK: i32.store8 0
; CHECK: i32.store8 0
@@ -3328,57 +3164,15 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_shorts_four_floats_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_shorts_four_floats_same_op:
-; CHECK: loop # label150:
-; CHECK: v128.load 0:p2align=1
-; CHECK: v128.load 16:p2align=1
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: v128.load 0:p2align=1
-; CHECK: v128.load 16:p2align=1
-; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 2, 3, 10, 11, 18, 19, 26, 27, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 4, 5, 12, 13, 20, 21, 28, 29, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
-; CHECK: f32x4.convert_i32x4_s
-; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: loop # label154:
+; CHECK: i32x4.load16x4_s 0:p2align=1
; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 6, 7, 14, 15, 22, 23, 30, 31, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK: i32x4.extend_low_i16x8_s
+; CHECK: i32x4.load16x4_s 0:p2align=1
; CHECK: f32x4.convert_i32x4_s
; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 12, 13, 14, 15, 28, 29, 30, 31
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 48:p2align=2
-; CHECK: i8x16.shuffle 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 24, 25, 26, 27
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 32:p2align=2
-; CHECK: i8x16.shuffle 4, 5, 6, 7, 20, 21, 22, 23, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
-; CHECK: v128.store 16:p2align=2
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 16, 17, 18, 19, 0, 1, 2, 3, 0, 1, 2, 3
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 16, 17, 18, 19
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label151:
+; CHECK: loop # label155:
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
@@ -3440,7 +3234,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_shorts_four_floats_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_shorts_four_floats_vary_op:
-; CHECK: loop # label154:
+; CHECK: loop # label158:
; CHECK: v128.load 0:p2align=1
; CHECK: v128.load 16:p2align=1
; CHECK: i8x16.shuffle 0, 1, 8, 9, 16, 17, 24, 25, 0, 1, 0, 1, 0, 1, 0, 1
@@ -3490,7 +3284,7 @@ define hidden void @four_shorts_four_floats_vary_op(ptr noundef readonly capture
; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31
; CHECK: v128.store 0:p2align=2
; CHECK: end_loop
-; CHECK: loop # label155:
+; CHECK: loop # label159:
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
; CHECK: i32.load16_s 0
@@ -3552,8 +3346,17 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_floats_four_shorts_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_shorts_same_op:
-; CHECK-NOT: v128.load
-; CHECK: loop # label157:
+; CHECK: loop # label162:
+; CHECK: v128.load 0:p2align=2
+; CHECK: v128.load 0:p2align=2
+; CHECK: f32x4.mul
+; CHECK: i32x4.trunc_sat_f32x4_s
+; CHECK: v128.const 65535, 65535, 65535, 65535
+; CHECK: v128.and
+; CHECK: i16x8.narrow_i32x4_u
+; CHECK: v128.store64_lane 0:p2align=1, 0
+; CHECK: end_loop
+; CHECK: loop # label163:
; CHECK: i32.store16 0
; CHECK: i32.store16 0
; CHECK: i32.store16 0
@@ -3607,8 +3410,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @four_floats_four_shorts_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
; CHECK-LABEL: four_floats_four_shorts_vary_op:
-; CHECK-NOT: v128.load
-; CHECK: loop # label159:
+; CHECK: loop # label165:
; CHECK: i32.store16 0
; CHECK: i32.store16 0
; CHECK: i32.store16 0
@@ -3662,8 +3464,7 @@ for.body: ; preds = %entry, %for.body
define hidden void @mac_3d_i8(ptr dead_on_unwind noalias writable writeonly sret(%struct.ThreeBytes) align 1 captures(none) %0, ptr noundef readonly captures(none) %1, ptr noundef readonly captures(none) %2, i32 noundef %3) {
; CHECK-LABEL: mac_3d_i8:
-; CHECK: loop # label162:
-; CHECK-NOT: v128.load
+; CHECK: loop # label168:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
@@ -3720,7 +3521,7 @@ define hidden void @mac_4d_i8(ptr dead_on_unwind noalias writable writeonly sret
; CHECK-LABEL: mac_4d_i8:
; CHECK: i32.store 0:p2align=0
; CHECK: v128.const 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK: loop # label166:
+; CHECK: loop # label172:
; CHECK: v128.load 0:p2align=0
; CHECK: v128.load 16:p2align=0
; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
@@ -3774,7 +3575,7 @@ define hidden void @mac_4d_i8(ptr dead_on_unwind noalias writable writeonly sret
; CHECK: i8x16.shuffle 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i8x16.add
; CHECK: i8x16.extract_lane_u 0
-; CHECK: loop # label167:
+; CHECK: loop # label173:
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
; CHECK: i32.load8_u 0
@@ -3849,7 +3650,7 @@ define hidden void @mac_2d_i64(ptr dead_on_unwind noalias writable sret(%struct.
; CHECK: v128.const 0, 0
; CHECK: i64x2.replace_lane 0
; CHECK: i64x2.replace_lane 0
-; CHECK: loop # label171:
+; CHECK: loop # label177:
; CHECK: v128.load 0:p2align=2
; CHECK: i8x16.shuffle 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3
; CHECK: i64x2.extend_low_i32x4_u
@@ -3871,7 +3672,7 @@ define hidden void @mac_2d_i64(ptr dead_on_unwind noalias writable sret(%struct.
; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
; CHECK: i64x2.add
; CHECK: i64x2.extract_lane 0
-; CHECK: loop # label172:
+; CHECK: loop # label178:
; CHECK: end_loop
entry:
%agg.result.promoted = load i64, ptr %agg.result, align 8
@@ -3924,7 +3725,7 @@ define hidden void @mac_4d_i64(ptr dead_on_unwind noalias writable sret(%struct.
; CHECK: i64x2.replace_lane 0
; CHECK: i64x2.replace_lane 0
; CHECK: i64x2.replace_lane 0
-; CHECK: loop # label176:
+; CHECK: loop # label182:
; CHECK: v128.load 0:p2align=2
; CHECK: v128.load 16:p2align=2
; CHECK: i8x16.shuffle 12, 13, 14, 15, 28, 29, 30, 31, 0, 1, 2, 3, 0, 1, 2, 3
@@ -3966,7 +3767,7 @@ define hidden void @mac_4d_i64(ptr dead_on_unwind noalias writable sret(%struct.
; CHECK: i8x16.shuffle 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7
; CHECK: i64x2.add
; CHECK: i64x2.extract_lane 0
-; CHECK: loop # label177:
+; CHECK: loop # label183:
; CHECK: end_loop
entry:
%agg.result.promoted = load i64, ptr %agg.result, align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll b/llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll
index 0aa9c976ac508..ed305477767e4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/foldshuffleidentity.ll
@@ -16,7 +16,7 @@ define dso_local void @f64_to_q15(ptr nofree noundef readonly captures(none) %pS
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP35]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP35]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP35]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP35]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP36:%.*]] = shl i64 [[N_VEC]], 5
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[PSRC]], i64 [[TMP36]]
@@ -31,37 +31,14 @@ define dso_local void @f64_to_q15(ptr nofree noundef readonly captures(none) %pS
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC]], i64 [[TMP42]]
; CHECK-NEXT: [[TMP43:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PDST]], i64 [[TMP43]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x double>, ptr [[NEXT_GEP]], align 8, !tbaa [[DOUBLE_TBAA7:![0-9]+]]
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <32 x double> [[WIDE_VEC]], <32 x double> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT: [[TMP10:%.*]] = fmul <8 x double> [[STRIDED_VEC]], splat (double 3.276800e+04)
-; CHECK-NEXT: [[TMP11:%.*]] = fptosi <8 x double> [[TMP10]] to <8 x i32>
-; CHECK-NEXT: [[TMP12:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP11]], <8 x i32> splat (i32 -32768))
-; CHECK-NEXT: [[TMP13:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP12]], <8 x i32> splat (i32 32767))
-; CHECK-NEXT: [[TMP14:%.*]] = trunc nsw <8 x i32> [[TMP13]] to <8 x i16>
-; CHECK-NEXT: [[TMP15:%.*]] = fmul <8 x double> [[STRIDED_VEC2]], splat (double 3.276800e+04)
-; CHECK-NEXT: [[TMP16:%.*]] = fptosi <8 x double> [[TMP15]] to <8 x i32>
-; CHECK-NEXT: [[TMP17:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP16]], <8 x i32> splat (i32 -32768))
-; CHECK-NEXT: [[TMP18:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP17]], <8 x i32> splat (i32 32767))
-; CHECK-NEXT: [[TMP19:%.*]] = trunc nsw <8 x i32> [[TMP18]] to <8 x i16>
-; CHECK-NEXT: [[TMP20:%.*]] = fmul <8 x double> [[STRIDED_VEC3]], splat (double 3.276800e+04)
-; CHECK-NEXT: [[TMP21:%.*]] = fptosi <8 x double> [[TMP20]] to <8 x i32>
-; CHECK-NEXT: [[TMP22:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP21]], <8 x i32> splat (i32 -32768))
-; CHECK-NEXT: [[TMP23:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP22]], <8 x i32> splat (i32 32767))
-; CHECK-NEXT: [[TMP24:%.*]] = trunc nsw <8 x i32> [[TMP23]] to <8 x i16>
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <8 x double>, ptr [[NEXT_GEP]], align 8, !tbaa [[DOUBLE_TBAA7:![0-9]+]]
; CHECK-NEXT: [[TMP25:%.*]] = fmul <8 x double> [[STRIDED_VEC4]], splat (double 3.276800e+04)
; CHECK-NEXT: [[TMP26:%.*]] = fptosi <8 x double> [[TMP25]] to <8 x i32>
; CHECK-NEXT: [[TMP27:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP26]], <8 x i32> splat (i32 -32768))
; CHECK-NEXT: [[TMP28:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP27]], <8 x i32> splat (i32 32767))
; CHECK-NEXT: [[TMP29:%.*]] = trunc nsw <8 x i32> [[TMP28]] to <8 x i16>
-; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <8 x i16> [[TMP14]], <8 x i16> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <8 x i16> [[TMP24]], <8 x i16> [[TMP29]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <16 x i16> [[TMP30]], <16 x i16> [[TMP31]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i16> [[TMP32]], <32 x i16> poison, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
-; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP1]], align 2, !tbaa [[SHORT_TBAA9:![0-9]+]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: store <8 x i16> [[TMP29]], ptr [[NEXT_GEP1]], align 2, !tbaa [[SHORT_TBAA9:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[WHILE_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -69,13 +46,13 @@ define dso_local void @f64_to_q15(ptr nofree noundef readonly captures(none) %pS
; CHECK-NEXT: br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP37]], %[[MIDDLE_BLOCK]] ], [ [[PSRC]], %[[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP39]], %[[MIDDLE_BLOCK]] ], [ [[SHR]], %[[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[PDST]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP39]], %[[MIDDLE_BLOCK]] ], [ [[SHR]], %[[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[PDST]], %[[WHILE_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[WHILE_BODY1:.*]]
; CHECK: [[WHILE_BODY1]]:
; CHECK-NEXT: [[PIN_068:%.*]] = phi ptr [ [[INCDEC_PTR15:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[BLKCNT_067:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[PDST_ADDR_066:%.*]] = phi ptr [ [[INCDEC_PTR20:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BLKCNT_067:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[PDST_ADDR_066:%.*]] = phi ptr [ [[INCDEC_PTR20:%.*]], %[[WHILE_BODY1]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PIN_068]], i64 8
; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[PIN_068]], align 8, !tbaa [[DOUBLE_TBAA7]]
; CHECK-NEXT: [[MUL:%.*]] = fmul double [[TMP0]], 3.276800e+04
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
index 9f21ec1806703..8f98aad7a49b6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
@@ -36,12 +36,12 @@ define void @interleave_groups_separated_by_offset(ptr %A, i64 %offset) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 16
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: store <32 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: store <32 x i8> zeroinitializer, ptr [[NEXT_GEP11]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1
+; CHECK-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP11]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 992
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
index 916cba1609221..3e2a20056bed0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
@@ -45,10 +45,10 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_16-LABEL: 'i8_factor_2'
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 1 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 1 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 1 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 1 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -92,16 +92,16 @@ define void @i16_factor_2(ptr %data, i64 %n) {
; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_8-LABEL: 'i16_factor_2'
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 1 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 1 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 1 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 1 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'i16_factor_2'
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 2 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -131,22 +131,22 @@ define void @i32_factor_2(ptr %data, i64 %n) {
; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_4-LABEL: 'i32_factor_2'
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 1 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 1 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_4: Cost of 1 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 1 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_8-LABEL: 'i32_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'i32_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -176,22 +176,22 @@ define void @i64_factor_2(ptr %data, i64 %n) {
; VF_2: Cost of 1 for VF 2: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_4-LABEL: 'i64_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_4: Cost of 2 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_8-LABEL: 'i64_factor_2'
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'i64_factor_2'
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
index 345dbaf761539..bc751d8c061ee 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
@@ -11,41 +11,27 @@ define void @test_complex_add_float(ptr %res, ptr noalias %A, ptr noalias %B, i6
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[TMP1]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[GEP_A_0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[WIDE_VEC5:%.*]] = load <8 x float>, ptr [[GEP_B_0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <8 x float> [[WIDE_VEC5]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <8 x float> [[WIDE_VEC5]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[WIDE_VEC8:%.*]] = load <8 x float>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[TMP6:%.*]] = fadd <4 x float> [[STRIDED_VEC]], [[STRIDED_VEC6]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd <4 x float> [[STRIDED_VEC3]], [[STRIDED_VEC9]]
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <4 x float>, ptr [[GEP_A_0]], align 4
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = load <4 x float>, ptr [[GEP_B_0]], align 4
+; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x float> [[STRIDED_VEC1]], [[STRIDED_VEC7]]
; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x float> [[STRIDED_VEC4]], [[STRIDED_VEC10]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC11:%.*]] = shufflevector <8 x float> [[TMP13]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC11]], ptr [[TMP11]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[TMP10]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP9]], ptr [[TMP11]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
index 18e536178189e..42397b3c7d877 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
@@ -81,7 +81,7 @@ define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
; VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
; VF4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF4: [[VECTOR_PH]]:
-; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2
; VF4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; VF4-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 4
; VF4-NEXT: [[TMP4:%.*]] = add i64 16, [[TMP3]]
@@ -91,14 +91,10 @@ define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
; VF4-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 4
; VF4-NEXT: [[OFFSET_IDX:%.*]] = add i64 16, [[TMP5]]
; VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP6]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
; VF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF4-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP8]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x double> [[WIDE_LOAD]], ptr [[TMP7]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VF4-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -165,11 +161,11 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 4
; VF2-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; VF2-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[NEXT_GEP]], align 8
-; VF2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[NEXT_GEP7]], align 8
+; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[NEXT_GEP]], align 8, !alias.scope [[META4:![0-9]+]]
+; VF2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[NEXT_GEP7]], align 8, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VF2-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
; VF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; VF2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -215,13 +211,13 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF2IC2-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP13]]
; VF2IC2-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX]]
; VF2IC2-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP13]]
-; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[NEXT_GEP]], align 8
-; VF2IC2-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x double>, ptr [[NEXT_GEP6]], align 8
-; VF2IC2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[NEXT_GEP8]], align 8
-; VF2IC2-NEXT: store <2 x double> [[WIDE_LOAD10]], ptr [[NEXT_GEP9]], align 8
+; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[NEXT_GEP]], align 8, !alias.scope [[META4:![0-9]+]]
+; VF2IC2-NEXT: [[WIDE_LOAD9:%.*]] = load <2 x double>, ptr [[NEXT_GEP6]], align 8, !alias.scope [[META4]]
+; VF2IC2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[NEXT_GEP8]], align 8, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
+; VF2IC2-NEXT: store <2 x double> [[WIDE_LOAD9]], ptr [[NEXT_GEP9]], align 8, !alias.scope [[META7]], !noalias [[META4]]
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VF2IC2-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF2IC2-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; VF2IC2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -253,7 +249,7 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF4-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; VF4-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; VF4: [[VECTOR_PH]]:
-; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
; VF4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; VF4-NEXT: [[TMP9:%.*]] = shl i64 [[N_VEC]], 4
; VF4-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
@@ -264,13 +260,9 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF4-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 4
; VF4-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; VF4-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[NEXT_GEP]], align 8, !alias.scope [[META4:![0-9]+]]
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP13:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP6]], align 8, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[NEXT_GEP]], align 8, !alias.scope [[META4:![0-9]+]]
+; VF4-NEXT: store <4 x double> [[WIDE_LOAD]], ptr [[NEXT_GEP6]], align 8, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VF4-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -315,7 +307,7 @@ define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
; VF2-NEXT: store <2 x i64> [[TMP3]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
-; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
; VF2-NEXT: br label %[[EXIT:.*]]
; VF2: [[EXIT]]:
@@ -341,7 +333,7 @@ define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
; VF2IC2-NEXT: store <2 x i64> [[TMP6]], ptr [[TMP4]], align 8
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
-; VF2IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF2IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2-NEXT: br label %[[EXIT:.*]]
; VF2IC2: [[EXIT]]:
@@ -357,15 +349,10 @@ define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET_IDX]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 1)
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP1]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
; VF4-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
index f0b026e7f43a8..e3fcb9418a72c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
@@ -9,12 +9,14 @@ target triple = "arm64-apple-macosx"
define void @no_epilogue_when_narrowed_covers_all(ptr %p) {
; CHECK-LABEL: define void @no_epilogue_when_narrowed_covers_all(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 2
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 4
@@ -31,8 +33,35 @@ define void @no_epilogue_when_narrowed_covers_all(ptr %p) {
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 500, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX1]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP9]]
+; CHECK-NEXT: store <2 x i64> splat (i64 1), ptr [[TMP10]], align 8
+; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 500
+; CHECK-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 true, label %[[EXIT1]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ 1000, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1000, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL3]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i64, ptr [[P0]], i64 1
+; CHECK-NEXT: store i64 1, ptr [[P0]], align 8
+; CHECK-NEXT: store i64 1, ptr [[P1]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -57,8 +86,88 @@ exit:
; (e.g. mul to shl) by later recipe simplification passes.
define fastcc i32 @narrow_interleave_epilogue_unknown_tc(ptr %end) {
; CHECK-LABEL: define fastcc i32 @narrow_interleave_epilogue_unknown_tc(
-; CHECK: vector.body:
-; CHECK: ret i32 0
+; CHECK-SAME: ptr [[END:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 15
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP2]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr null, i64 [[TMP3]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP5]], 32
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP5]], 48
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr null, i64 [[TMP5]]
+; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP6]]
+; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr null, i64 [[TMP7]]
+; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr null, i64 [[TMP8]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[NEXT_GEP]], align 8
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <2 x i64>, ptr [[NEXT_GEP3]], align 8
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <2 x i64>, ptr [[NEXT_GEP4]], align 8
+; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <2 x i64>, ptr [[NEXT_GEP5]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = sdiv <2 x i64> [[WIDE_LOAD]], splat (i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = sdiv <2 x i64> [[WIDE_LOAD6]], splat (i64 4)
+; CHECK-NEXT: [[TMP11:%.*]] = sdiv <2 x i64> [[WIDE_LOAD7]], splat (i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = sdiv <2 x i64> [[WIDE_LOAD8]], splat (i64 4)
+; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[NEXT_GEP]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP10]], ptr [[NEXT_GEP3]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP11]], ptr [[NEXT_GEP4]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP12]], ptr [[NEXT_GEP5]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr null, i64 [[TMP14]]
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX9]], 4
+; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr null, i64 [[TMP16]]
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <2 x i64>, ptr [[NEXT_GEP10]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = sdiv <2 x i64> [[WIDE_LOAD11]], splat (i64 4)
+; CHECK-NEXT: store <2 x i64> [[TMP17]], ptr [[NEXT_GEP10]], align 8
+; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX9]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[TMP2]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP4]], %[[VEC_EPILOG_ITER_CHECK]] ], [ null, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[V0:%.*]] = load i64, ptr [[PTR]], align 8
+; CHECK-NEXT: [[DIV0:%.*]] = sdiv i64 [[V0]], 4
+; CHECK-NEXT: store i64 [[DIV0]], ptr [[PTR]], align 8
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 8
+; CHECK-NEXT: [[V1:%.*]] = load i64, ptr [[GEP1]], align 8
+; CHECK-NEXT: [[DIV1:%.*]] = sdiv i64 [[V1]], 4
+; CHECK-NEXT: store i64 [[DIV1]], ptr [[GEP1]], align 8
+; CHECK-NEXT: [[PTR_NEXT]] = getelementptr nusw i8, ptr [[PTR]], i64 16
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult ptr [[PTR]], [[END]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 0
;
entry:
br label %loop
@@ -85,7 +194,72 @@ exit:
%pair = type { i64, i64 }
define void @test(ptr noalias %A, i64 %v, i64 %n) #0 {
; CHECK-LABEL: define void @test(
-; CHECK: vector.body:
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[A]], i64 [[INDEX]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i64 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i64 [[TMP2]], i32 0
+; CHECK-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 8
+; CHECK-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 8
+; CHECK-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 8
+; CHECK-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF10:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i64 [[INDEX6]], i32 0
+; CHECK-NEXT: store <4 x i64> [[BROADCAST_SPLAT5]], ptr [[TMP8]], align 8
+; CHECK-NEXT: [[INDEX_NEXT7]] = add nuw i64 [[INDEX6]], 2
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT7]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N8]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i64 [[IV]], i32 0
+; CHECK-NEXT: [[IDX_1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i64 [[IV]], i32 1
+; CHECK-NEXT: store i64 [[V]], ptr [[IDX_0]], align 8
+; CHECK-NEXT: store i64 [[V]], ptr [[IDX_1]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
index 73fd53011e956..f1f9a6702c176 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
@@ -189,15 +189,15 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[INDEX]]
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
; VF2-NEXT: [[WIDE_VEC2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
-; VF2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x float> [[WIDE_VEC2]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x float> [[WIDE_VEC2]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
-; VF2-NEXT: [[TMP3:%.*]] = fadd <2 x float> [[STRIDED_VEC]], [[STRIDED_VEC3]]
+; VF2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x float> [[WIDE_VEC2]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <4 x float> [[WIDE_VEC2]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
; VF2-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[STRIDED_VEC1]], [[STRIDED_VEC4]]
+; VF2-NEXT: [[TMP3:%.*]] = fadd <2 x float> [[STRIDED_VEC2]], [[STRIDED_VEC5]]
; VF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[INDEX]]
-; VF2-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 4
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -215,18 +215,17 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
; VF4: [[VECTOR_PH]]:
; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
; VF4: [[VECTOR_BODY]]:
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[A]], align 4
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[B]], align 4
-; VF4-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[STRIDED_VEC]], [[STRIDED_VEC3]]
+; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[INDEX]]
+; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[INDEX]]
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; VF4-NEXT: [[STRIDED_VEC4:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; VF4-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[STRIDED_VEC1]], [[STRIDED_VEC4]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[RES]], align 4
-; VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; VF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[INDEX]]
+; VF4-NEXT: store <4 x float> [[TMP3]], ptr [[TMP5]], align 4
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
+; VF4-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
; VF4-NEXT: br label %[[EXIT:.*]]
; VF4: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll
index 6fd49e2e99f18..cef5caa9fdd70 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll
@@ -32,9 +32,9 @@ define i64 @test_wide_op_live_out_constant_store_group(ptr noalias %res, ptr noa
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RES]], i64 [[INDEX]]
-; VF4-NEXT: store <8 x i64> zeroinitializer, ptr [[TMP0]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; VF4-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; VF4-NEXT: store <4 x i64> zeroinitializer, ptr [[TMP0]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF4-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
; VF4-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
; VF4-NEXT: br label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll
index 5dd44bf04ca1a..ff30217dc4cba 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll
@@ -40,13 +40,13 @@ define void @test_2xi64_matching_zext_interleave_group(ptr noalias %dst, ptr %sr
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 8
+; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP2:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -112,13 +112,13 @@ define void @test_2xi64_matching_sext_interleave_group(ptr noalias %dst, ptr %sr
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 8
+; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -259,14 +259,14 @@ define void @test_2xi64_matching_cast_add_interleave_group(ptr noalias %dst, ptr
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 8
+; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP2:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP3:%.*]] = add <4 x i64> [[TMP2]], splat (i64 2)
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -415,14 +415,14 @@ define void @test_2xi64_add_cast_interleave_group(ptr noalias %dst, ptr %src) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 8
+; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 2)
; VF4-NEXT: [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
index 2b747bef56b72..8a1a716fa75aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
@@ -37,15 +37,10 @@ define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr no
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP1]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP2]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP3:%.*]] = fneg <4 x double> [[STRIDED_VEC]]
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
; VF4-NEXT: [[TMP4:%.*]] = fneg <4 x double> [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x double> [[TMP4]], ptr [[TMP2]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -191,18 +186,15 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor) {
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]]
-; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP2]], i64 0
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP10:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP10]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP11]], align 8
-; VF4-NEXT: [[TMP19:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[TMP41:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP20:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP19]]
+; VF4-NEXT: [[TMP41:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
; VF4-NEXT: [[TMP42:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP41]]
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP20]], <4 x i64> [[TMP42]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP11]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[TMP42]], ptr [[TMP11]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -981,16 +973,18 @@ define void @test_2xi64_sub_of_wide_loads(ptr noalias %data, ptr noalias %A, ptr
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
-; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP2]], i64 0
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
-; VF4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; VF4-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+; VF4-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; VF4-NEXT: [[WIDE_LOAD1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP5:%.*]] = sub <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; VF4-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP0]], 1
; VF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
-; VF4-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
+; VF4-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP7]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
; VF4-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -1224,18 +1218,13 @@ define void @multiple_store_groups_storing_same_wide_bin_op(ptr noalias %A, ptr
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP0]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP1:%.*]] = fadd contract <4 x double> [[STRIDED_VEC]], splat (double 2.000000e+01)
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = load <4 x double>, ptr [[TMP0]], align 8
; VF4-NEXT: [[TMP2:%.*]] = fadd contract <4 x double> [[STRIDED_VEC1]], splat (double 2.000000e+01)
; VF4-NEXT: [[TMP3:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP4]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
+; VF4-NEXT: store <4 x double> [[TMP2]], ptr [[TMP3]], align 8
; VF4-NEXT: [[TMP5:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x double> [[TMP2]], ptr [[TMP5]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -1304,16 +1293,11 @@ define void @flag_mismatch_disjoint(ptr noalias %dst, ptr noalias %src) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP2:%.*]] = or disjoint <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP4:%.*]] = or <4 x i64> [[STRIDED_VEC1]], splat (i64 1)
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP3]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -1544,17 +1528,12 @@ define void @test_2xdouble_same_fcmp_predicate(ptr noalias %data) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP2:%.*]] = fcmp ogt <4 x double> [[STRIDED_VEC]], zeroinitializer
-; VF4-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> zeroinitializer, <4 x double> [[STRIDED_VEC]]
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP4:%.*]] = fcmp ogt <4 x double> [[STRIDED_VEC1]], zeroinitializer
-; VF4-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x double> zeroinitializer, <4 x double> [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP6:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP4]], <4 x double> zeroinitializer, <4 x double> [[WIDE_LOAD1]]
+; VF4-NEXT: store <4 x double> [[TMP3]], ptr [[TMP1]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index 5c9a3eac44fa0..e3f4783b7d655 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -36,13 +36,9 @@ define void @load_store_interleave_group(ptr noalias %data) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[STRIDED_VEC]], <4 x i64> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; VF4-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -103,14 +99,10 @@ define void @load_store_interleave_group_different_objecs(ptr noalias %src, ptr
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP1]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP1]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[STRIDED_VEC]], <4 x i64> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP3]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -239,8 +231,8 @@ define void @same_constant_store_interleave_group(i64 %x, ptr noalias %dst) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: store <8 x i64> zeroinitializer, ptr [[TMP1]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -359,10 +351,8 @@ define void @same_live_in_store_interleave_group(i64 %x, ptr noalias %dst) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -495,10 +485,8 @@ define void @single_uniform_load_store_interleave_group(ptr noalias %src, ptr no
; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
; VF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -635,16 +623,13 @@ define void @multiple_store_groups_storing_same_load_group(ptr noalias %A, ptr n
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
-; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP0]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP0]], align 8
+; VF4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP0]], align 8
; VF4-NEXT: [[TMP1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF4-NEXT: store <4 x double> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP3:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
-; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: store <4 x double> [[WIDE_LOAD1]], ptr [[TMP3]], align 8
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF4-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
@@ -676,3 +661,165 @@ loop:
exit:
ret void
}
+
+define void @powi_two_fields_same_exponent(ptr noalias %src, ptr noalias %dst, i64 %n) {
+; VF2-LABEL: define void @powi_two_fields_same_exponent(
+; VF2-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; VF2-NEXT: [[ENTRY:.*:]]
+; VF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; VF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2: [[VECTOR_PH]]:
+; VF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; VF2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF2: [[VECTOR_BODY]]:
+; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[SRC]], i64 [[INDEX]], i32 0
+; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; VF2-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[POWI:%.*]] = call <2 x float> @llvm.powi.v2f32.i32(<2 x float> [[WIDE_LOAD]], i32 2)
+; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[DST]], i64 [[INDEX]], i32 0
+; VF2-NEXT: [[TMP3:%.*]] = call <2 x float> @llvm.powi.v2f32.i32(<2 x float> [[STRIDED_VEC1]], i32 2)
+; VF2-NEXT: [[TMP4:%.*]] = shufflevector <2 x float> [[POWI]], <2 x float> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
+; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VF2: [[MIDDLE_BLOCK]]:
+; VF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2: [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @powi_two_fields_same_exponent(
+; VF4-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; VF4-NEXT: [[ENTRY:.*:]]
+; VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; VF4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4: [[VECTOR_PH]]:
+; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; VF4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4: [[VECTOR_BODY]]:
+; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[SRC]], i64 [[INDEX]], i32 0
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; VF4-NEXT: [[TMP1:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[WIDE_LOAD]], i32 2)
+; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[DST]], i64 [[INDEX]], i32 0
+; VF4-NEXT: store <4 x float> [[TMP1]], ptr [[TMP2]], align 4
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF4-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF4-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VF4: [[MIDDLE_BLOCK]]:
+; VF4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF4-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src.0 = getelementptr inbounds { float, float }, ptr %src, i64 %iv, i32 0
+ %x0 = load float, ptr %src.0, align 4
+ %p0 = call float @llvm.powi.f32.i32(float %x0, i32 2)
+ %dst.0 = getelementptr inbounds { float, float }, ptr %dst, i64 %iv, i32 0
+ store float %p0, ptr %dst.0, align 4
+ %src.1 = getelementptr inbounds { float, float }, ptr %src, i64 %iv, i32 1
+ %x1 = load float, ptr %src.1, align 4
+ %p1 = call float @llvm.powi.f32.i32(float %x1, i32 2)
+ %dst.1 = getelementptr inbounds { float, float }, ptr %dst, i64 %iv, i32 1
+ store float %p1, ptr %dst.1, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+
+define void @powi_two_fields(ptr noalias %src, ptr noalias %dst, i64 %n) {
+; VF2-LABEL: define void @powi_two_fields(
+; VF2-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; VF2-NEXT: [[ENTRY:.*:]]
+; VF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; VF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2: [[VECTOR_PH]]:
+; VF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; VF2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF2: [[VECTOR_BODY]]:
+; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[SRC]], i64 [[INDEX]], i32 0
+; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[POWI0:%.*]] = call <2 x float> @llvm.powi.v2f32.i32(<2 x float> [[STRIDED_VEC]], i32 2)
+; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[DST]], i64 [[INDEX]], i32 0
+; VF2-NEXT: [[POWI1:%.*]] = call <2 x float> @llvm.powi.v2f32.i32(<2 x float> [[STRIDED_VEC1]], i32 3)
+; VF2-NEXT: [[COMBINED:%.*]] = shufflevector <2 x float> [[POWI0]], <2 x float> [[POWI1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[COMBINED]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
+; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF2: [[MIDDLE_BLOCK]]:
+; VF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2: [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @powi_two_fields(
+; VF4-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; VF4-NEXT: [[ENTRY:.*:]]
+; VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; VF4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4: [[VECTOR_PH]]:
+; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; VF4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4: [[VECTOR_BODY]]:
+; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[SRC]], i64 [[INDEX]], i32 0
+; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP0]], align 4
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[POWI0:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[STRIDED_VEC]], i32 2)
+; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[DST]], i64 [[INDEX]], i32 0
+; VF4-NEXT: [[POWI1:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[STRIDED_VEC1]], i32 3)
+; VF4-NEXT: [[COMBINED:%.*]] = shufflevector <4 x float> [[POWI0]], <4 x float> [[POWI1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[COMBINED]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF4-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF4: [[MIDDLE_BLOCK]]:
+; VF4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF4-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %src.0 = getelementptr inbounds { float, float }, ptr %src, i64 %iv, i32 0
+ %x0 = load float, ptr %src.0, align 4
+ %p0 = call float @llvm.powi.f32.i32(float %x0, i32 2)
+ %dst.0 = getelementptr inbounds { float, float }, ptr %dst, i64 %iv, i32 0
+ store float %p0, ptr %dst.0, align 4
+ %src.1 = getelementptr inbounds { float, float }, ptr %src, i64 %iv, i32 1
+ %x1 = load float, ptr %src.1, align 4
+ %p1 = call float @llvm.powi.f32.i32(float %x1, i32 3)
+ %dst.1 = getelementptr inbounds { float, float }, ptr %dst, i64 %iv, i32 1
+ store float %p1, ptr %dst.1, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare float @llvm.powi.f32.i32(float, i32)
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
index 3c7f19944449e..1987f328957e5 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
@@ -35,10 +35,10 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_16-LABEL: 'i8_factor_2'
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 2 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 1 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 1 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 1 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 1 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -74,16 +74,16 @@ define void @i16_factor_2(ptr %data, i64 %n) {
; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_8-LABEL: 'i16_factor_2'
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 2 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 1 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 1 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 1 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 1 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'i16_factor_2'
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 2 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -113,22 +113,22 @@ define void @i32_factor_2(ptr %data, i64 %n) {
; VF_2: Cost of 2 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_4-LABEL: 'i32_factor_2'
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 2 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 1 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 1 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_4: Cost of 1 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 1 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_8-LABEL: 'i32_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'i32_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -164,16 +164,16 @@ define void @half_factor_2(ptr %data, i64 %n) {
; VF_4: Cost of 40 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_8-LABEL: 'half_factor_2'
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 8 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 8 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'half_factor_2'
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 16 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 16 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
index 4c19c36f232bb..55da6a6777750 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
@@ -39,10 +39,10 @@ define void @i8_factor_2(ptr %data, i64 %n) #0 {
; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_16-LABEL: 'i8_factor_2'
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 2 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 2 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -86,16 +86,16 @@ define void @i16_factor_2(ptr %data, i64 %n) #0 {
; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_8-LABEL: 'i16_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'i16_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -133,22 +133,22 @@ define void @i32_factor_2(ptr %data, i64 %n) #0 {
; VF_2: Cost of 4 for VF 2: REPLICATE store ir<%tmp3>, ir<%tmp1>
;
; VF_4-LABEL: 'i32_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_4: Cost of 2 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_8-LABEL: 'i32_factor_2'
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'i32_factor_2'
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -269,16 +269,16 @@ define void @f16_factor_2(ptr %data, i64 %n) #0 {
; VF_4: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_8-LABEL: 'f16_factor_2'
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 4 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'f16_factor_2'
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -308,22 +308,22 @@ define void @f32_factor_2(ptr %data, i64 %n) #0 {
; VF_2: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
;
; VF_4-LABEL: 'f32_factor_2'
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_4: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_4: Cost of 2 for VF 4: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_4: Cost of 2 for VF 4: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_8-LABEL: 'f32_factor_2'
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_8: Cost of 8 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp2>
;
; VF_16-LABEL: 'f32_factor_2'
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
-; VF_16: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp2> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp2>
;
entry:
br label %for.body
@@ -1617,16 +1617,16 @@ define void @f16_factor_4(ptr %data, i64 %n) #0 {
; VF_4: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
;
; VF_8-LABEL: 'f16_factor_4'
-; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 72 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_8: Cost of 2 for VF 8: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 2 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp4>
;
; VF_16-LABEL: 'f16_factor_4'
-; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 144 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_16: Cost of 4 for VF 16: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 4 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp4>
;
entry:
br label %for.body
@@ -1690,16 +1690,16 @@ define void @f32_factor_4(ptr %data, i64 %n) #0 {
; VF_4: Cost of 8 for VF 4: WIDEN store ir<%tmp3>, ir<%tmp7>
;
; VF_8-LABEL: 'f32_factor_4'
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_8: Cost of 80 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_8: Cost of 4 for VF 8: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_8: Cost of 4 for VF 8: WIDEN store ir<%tmp0>, ir<%tmp4>
;
; VF_16-LABEL: 'f32_factor_4'
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
-; VF_16: Cost of 160 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp4>
+; VF_16: Cost of 8 for VF 16: WIDEN ir<%tmp4> = load ir<%tmp0>
+; VF_16: Cost of 8 for VF 16: WIDEN store ir<%tmp0>, ir<%tmp4>
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index cc65d7903c5a7..ccd632f4e7d1a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -44,15 +44,10 @@ define void @load_store_factor2_i32(ptr %p) {
; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; FIXED-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 1
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
-; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT: [[TMP2:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; FIXED-NEXT: [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; FIXED-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; FIXED-NEXT: store <16 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
-; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
+; FIXED-NEXT: [[TMP2:%.*]] = add <8 x i32> [[WIDE_LOAD]], <i32 1, i32 2, i32 1, i32 2, i32 1, i32 2, i32 1, i32 2>
+; FIXED-NEXT: store <8 x i32> [[TMP2]], ptr [[TMP1]], align 4
+; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; FIXED: middle.block:
@@ -156,15 +151,10 @@ define void @load_store_factor2_i64(ptr %p) {
; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; FIXED-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 1
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT: [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT: [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; FIXED-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; FIXED-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], <i64 1, i64 2, i64 1, i64 2>
+; FIXED-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP1]], align 8
+; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; FIXED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; FIXED: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
index 1b0a14ef9e556..50c0de706f899 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
@@ -124,8 +124,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
; FIXED-OPT-NF2-LABEL: 'i8_factor_2'
; FIXED-OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
@@ -136,8 +134,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
; FIXED-OPT-NF3-LABEL: 'i8_factor_2'
; FIXED-OPT-NF3: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
@@ -148,8 +144,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-OPT-NF3: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF3: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF3: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF3: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF3: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
; FIXED-OPT-NF4-LABEL: 'i8_factor_2'
; FIXED-OPT-NF4: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
@@ -160,8 +154,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-OPT-NF4: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF4: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF4: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF4: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF4: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
; FIXED-OPT-NF5-LABEL: 'i8_factor_2'
; FIXED-OPT-NF5: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
@@ -172,8 +164,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-OPT-NF5: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF5: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF5: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF5: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF5: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
; FIXED-OPT-NF6-LABEL: 'i8_factor_2'
; FIXED-OPT-NF6: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
@@ -184,8 +174,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-OPT-NF6: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF6: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF6: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF6: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF6: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
; FIXED-OPT-NF7-LABEL: 'i8_factor_2'
; FIXED-OPT-NF7: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
@@ -196,8 +184,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-OPT-NF7: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF7: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF7: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF7: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF7: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
; FIXED-OPT-NF8-LABEL: 'i8_factor_2'
; FIXED-OPT-NF8: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2, ir<%p0>
@@ -208,8 +194,6 @@ define void @i8_factor_2(ptr %data, i64 %n) {
; FIXED-OPT-NF8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF8: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
; FIXED-OPT-NF8: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF8: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
-; FIXED-OPT-NF8: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2, ir<%p0>
;
entry:
br label %for.body
@@ -556,8 +540,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
; FIXED-OPT-NF2-LABEL: 'i8_factor_4'
; FIXED-OPT-NF2: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
@@ -568,8 +550,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-OPT-NF2: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF2: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF2: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF2: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF2: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
; FIXED-OPT-NF3-LABEL: 'i8_factor_4'
; FIXED-OPT-NF3: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
@@ -580,8 +560,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-OPT-NF3: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF3: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF3: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF3: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF3: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
; FIXED-OPT-NF4-LABEL: 'i8_factor_4'
; FIXED-OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
@@ -592,8 +570,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
; FIXED-OPT-NF5-LABEL: 'i8_factor_4'
; FIXED-OPT-NF5: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
@@ -604,8 +580,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-OPT-NF5: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF5: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF5: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF5: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF5: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
; FIXED-OPT-NF6-LABEL: 'i8_factor_4'
; FIXED-OPT-NF6: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
@@ -616,8 +590,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-OPT-NF6: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF6: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF6: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF6: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF6: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
; FIXED-OPT-NF7-LABEL: 'i8_factor_4'
; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
@@ -628,8 +600,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-OPT-NF7: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF7: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF7: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF7: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF7: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
; FIXED-OPT-NF8-LABEL: 'i8_factor_4'
; FIXED-OPT-NF8: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4, ir<%p0>
@@ -640,8 +610,6 @@ define void @i8_factor_4(ptr %data, i64 %n) {
; FIXED-OPT-NF8: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF8: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
; FIXED-OPT-NF8: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF8: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
-; FIXED-OPT-NF8: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4, ir<%p0>
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
index 2c9dfec8c1910..75308f968524e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
@@ -280,54 +280,54 @@ define void @load_store_interleave_group_i32(ptr noalias %data) {
;
; EPILOGUE-LABEL: define void @load_store_interleave_group_i32(
; EPILOGUE-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0]] {
-; EPILOGUE-NEXT: [[ENTRY:.*]]:
-; EPILOGUE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; EPILOGUE-NEXT: [[VECTOR_PH:.*]]:
+; EPILOGUE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP3]], 2
; EPILOGUE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[UMAX]]
-; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; EPILOGUE: [[VECTOR_PH]]:
+; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY:.*]]
+; EPILOGUE: [[VECTOR_BODY]]:
; EPILOGUE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP2]]
; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
-; EPILOGUE: [[VECTOR_BODY]]:
-; EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; EPILOGUE-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 2
-; EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP4]]
-; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 8
-; EPILOGUE-NEXT: store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP5]], align 8
-; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; EPILOGUE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; EPILOGUE: [[MIDDLE_BLOCK]]:
-; EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
-; EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; EPILOGUE: [[SCALAR_PH]]:
-; EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; EPILOGUE-NEXT: br label %[[LOOP:.*]]
; EPILOGUE: [[LOOP]]:
-; EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; EPILOGUE-NEXT: [[MUL_2:%.*]] = shl nsw i64 [[IV]], 2
-; EPILOGUE-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[MUL_2]]
-; EPILOGUE-NEXT: [[L_0:%.*]] = load i32, ptr [[DATA_0]], align 8
-; EPILOGUE-NEXT: store i32 [[L_0]], ptr [[DATA_0]], align 8
-; EPILOGUE-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1
+; EPILOGUE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[MUL_2]]
+; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 8
+; EPILOGUE-NEXT: store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP4]], align 8
+; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP2]]
+; EPILOGUE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; EPILOGUE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; EPILOGUE: [[MIDDLE_BLOCK]]:
+; EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
+; EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT1:.*]], label %[[SCALAR_PH]]
+; EPILOGUE: [[SCALAR_PH]]:
+; EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_PH]] ]
+; EPILOGUE-NEXT: br label %[[LOOP1:.*]]
+; EPILOGUE: [[LOOP1]]:
+; EPILOGUE-NEXT: [[TMP0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; EPILOGUE-NEXT: [[ADD_3:%.*]] = shl nsw i64 [[TMP0]], 2
+; EPILOGUE-NEXT: [[DATA_3:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_3]]
+; EPILOGUE-NEXT: [[L_0:%.*]] = load i32, ptr [[DATA_3]], align 8
+; EPILOGUE-NEXT: store i32 [[L_0]], ptr [[DATA_3]], align 8
+; EPILOGUE-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[ADD_3]], 1
; EPILOGUE-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_1]]
; EPILOGUE-NEXT: [[L_1:%.*]] = load i32, ptr [[DATA_1]], align 8
; EPILOGUE-NEXT: store i32 [[L_1]], ptr [[DATA_1]], align 8
-; EPILOGUE-NEXT: [[ADD_2:%.*]] = add i64 [[MUL_2]], 2
+; EPILOGUE-NEXT: [[ADD_2:%.*]] = add i64 [[ADD_3]], 2
; EPILOGUE-NEXT: [[DATA_2:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_2]]
; EPILOGUE-NEXT: [[L_2:%.*]] = load i32, ptr [[DATA_2]], align 8
; EPILOGUE-NEXT: store i32 [[L_2]], ptr [[DATA_2]], align 8
-; EPILOGUE-NEXT: [[ADD_3:%.*]] = add i64 [[MUL_2]], 3
-; EPILOGUE-NEXT: [[DATA_3:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_3]]
-; EPILOGUE-NEXT: [[L_3:%.*]] = load i32, ptr [[DATA_3]], align 8
-; EPILOGUE-NEXT: store i32 [[L_3]], ptr [[DATA_3]], align 8
-; EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; EPILOGUE-NEXT: [[ADD_4:%.*]] = add i64 [[ADD_3]], 3
+; EPILOGUE-NEXT: [[DATA_4:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_4]]
+; EPILOGUE-NEXT: [[L_3:%.*]] = load i32, ptr [[DATA_4]], align 8
+; EPILOGUE-NEXT: store i32 [[L_3]], ptr [[DATA_4]], align 8
+; EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[TMP0]], 1
; EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; EPILOGUE: [[EXIT]]:
+; EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; EPILOGUE: [[EXIT1]]:
; EPILOGUE-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
index d123a62d443cc..801b0e33b572b 100644
--- a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
@@ -32,16 +32,7 @@ define hidden void @two_ints_same_op(ptr noalias nocapture noundef writeonly %0,
; CHECK: store ir<%13> to index 0
; CHECK: store ir<%19> to index 1
; CHECK: Cost for VF 2: 27 (Estimated cost per lane: 13.5)
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%9>
-; CHECK: ir<%10> = load from index 0
-; CHECK: ir<%16> = load from index 1
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%11>
-; CHECK: ir<%12> = load from index 0
-; CHECK: ir<%18> = load from index 1
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%14>
-; CHECK: store ir<%13> to index 0
-; CHECK: store ir<%19> to index 1
-; CHECK: Cost for VF 4: 24 (Estimated cost per lane: 6)
+; CHECK: Cost for VF 4: 11 (Estimated cost per lane: 2.75)
; CHECK: LV: Selecting VF: 4.
;
%5 = icmp eq i32 %3, 0
@@ -286,23 +277,8 @@ define hidden void @four_shorts_same_op(ptr noalias nocapture noundef writeonly
; CHECK: store ir<%25> to index 2
; CHECK: store ir<%31> to index 3
; CHECK: Cost for VF 4: 62 (Estimated cost per lane: 15.5)
-; CHECK: Cost of 68 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%9>
-; CHECK: ir<%10> = load from index 0
-; CHECK: ir<%16> = load from index 1
-; CHECK: ir<%22> = load from index 2
-; CHECK: ir<%28> = load from index 3
-; CHECK: Cost of 68 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%11>
-; CHECK: ir<%12> = load from index 0
-; CHECK: ir<%18> = load from index 1
-; CHECK: ir<%24> = load from index 2
-; CHECK: ir<%30> = load from index 3
-; CHECK: Cost of 68 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%14>
-; CHECK: store ir<%13> to index 0
-; CHECK: store ir<%19> to index 1
-; CHECK: store ir<%25> to index 2
-; CHECK: store ir<%31> to index 3
-; CHECK: Cost for VF 8: 212 (Estimated cost per lane: 26.5)
-; CHECK: LV: Selecting VF: 4.
+; CHECK: Cost for VF 8: 11 (Estimated cost per lane: 1.38)
+; CHECK: LV: Selecting VF: 8.
;
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -689,16 +665,7 @@ define hidden void @two_bytes_same_op(ptr noalias nocapture noundef writeonly %0
; CHECK: store ir<%13> to index 0
; CHECK: store ir<%19> to index 1
; CHECK: Cost for VF 8: 33 (Estimated cost per lane: 4.13)
-; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%9>
-; CHECK: ir<%10> = load from index 0
-; CHECK: ir<%16> = load from index 1
-; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%11>
-; CHECK: ir<%12> = load from index 0
-; CHECK: ir<%18> = load from index 1
-; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, ir<%14>
-; CHECK: store ir<%13> to index 0
-; CHECK: store ir<%19> to index 1
-; CHECK: Cost for VF 16: 30 (Estimated cost per lane: 1.88)
+; CHECK: Cost for VF 16: 14 (Estimated cost per lane: 0.875)
; CHECK: LV: Selecting VF: 16.
;
%5 = icmp eq i32 %3, 0
@@ -1003,23 +970,8 @@ define hidden void @four_bytes_same_op(ptr noalias nocapture noundef writeonly %
; CHECK: store ir<%25> to index 2
; CHECK: store ir<%31> to index 3
; CHECK: Cost for VF 8: 86 (Estimated cost per lane: 10.8)
-; CHECK: Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%9>
-; CHECK: ir<%10> = load from index 0
-; CHECK: ir<%16> = load from index 1
-; CHECK: ir<%22> = load from index 2
-; CHECK: ir<%28> = load from index 3
-; CHECK: Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%11>
-; CHECK: ir<%12> = load from index 0
-; CHECK: ir<%18> = load from index 1
-; CHECK: ir<%24> = load from index 2
-; CHECK: ir<%30> = load from index 3
-; CHECK: Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%14>
-; CHECK: store ir<%13> to index 0
-; CHECK: store ir<%19> to index 1
-; CHECK: store ir<%25> to index 2
-; CHECK: store ir<%31> to index 3
-; CHECK: Cost for VF 16: 404 (Estimated cost per lane: 25.3)
-; CHECK: LV: Selecting VF: 8.
+; CHECK: Cost for VF 16: 11 (Estimated cost per lane: 0.688)
+; CHECK: LV: Selecting VF: 16.
;
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -1365,35 +1317,8 @@ define hidden void @eight_bytes_same_op(ptr noalias nocapture noundef writeonly
; CHECK: store ir<%49> to index 6
; CHECK: store ir<%55> to index 7
; CHECK: Cost for VF 8: 432 (Estimated cost per lane: 54)
-; CHECK: Cost of 264 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%9>
-; CHECK: ir<%10> = load from index 0
-; CHECK: ir<%16> = load from index 1
-; CHECK: ir<%22> = load from index 2
-; CHECK: ir<%28> = load from index 3
-; CHECK: ir<%34> = load from index 4
-; CHECK: ir<%40> = load from index 5
-; CHECK: ir<%46> = load from index 6
-; CHECK: ir<%52> = load from index 7
-; CHECK: Cost of 264 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%11>
-; CHECK: ir<%12> = load from index 0
-; CHECK: ir<%18> = load from index 1
-; CHECK: ir<%24> = load from index 2
-; CHECK: ir<%30> = load from index 3
-; CHECK: ir<%36> = load from index 4
-; CHECK: ir<%42> = load from index 5
-; CHECK: ir<%48> = load from index 6
-; CHECK: ir<%54> = load from index 7
-; CHECK: Cost of 264 for VF 16: INTERLEAVE-GROUP with factor 8, ir<%14>
-; CHECK: store ir<%13> to index 0
-; CHECK: store ir<%19> to index 1
-; CHECK: store ir<%25> to index 2
-; CHECK: store ir<%31> to index 3
-; CHECK: store ir<%37> to index 4
-; CHECK: store ir<%43> to index 5
-; CHECK: store ir<%49> to index 6
-; CHECK: store ir<%55> to index 7
-; CHECK: Cost for VF 16: 828 (Estimated cost per lane: 51.8)
-; CHECK: LV: Selecting VF: 1.
+; CHECK: Cost for VF 16: 14 (Estimated cost per lane: 0.875)
+; CHECK: LV: Selecting VF: 16.
;
%5 = icmp eq i32 %3, 0
br i1 %5, label %6, label %7
@@ -1858,27 +1783,7 @@ define hidden void @four_bytes_into_four_ints_same_op(ptr noalias nocapture noun
; CHECK: store ir<%38> to index 2
; CHECK: store ir<%48> to index 3
; CHECK: Cost for VF 2: 89 (Estimated cost per lane: 44.5)
-; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%9>
-; CHECK: ir<%10> = load from index 0
-; CHECK: ir<%20> = load from index 1
-; CHECK: ir<%30> = load from index 2
-; CHECK: ir<%40> = load from index 3
-; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%12>
-; CHECK: ir<%13> = load from index 0
-; CHECK: ir<%23> = load from index 1
-; CHECK: ir<%33> = load from index 2
-; CHECK: ir<%43> = load from index 3
-; CHECK: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%16>
-; CHECK: ir<%17> = load from index 0
-; CHECK: ir<%27> = load from index 1
-; CHECK: ir<%37> = load from index 2
-; CHECK: ir<%47> = load from index 3
-; CHECK: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%16>
-; CHECK: store ir<%18> to index 0
-; CHECK: store ir<%28> to index 1
-; CHECK: store ir<%38> to index 2
-; CHECK: store ir<%48> to index 3
-; CHECK: Cost for VF 4: 104 (Estimated cost per lane: 26)
+; CHECK: Cost for VF 4: 16 (Estimated cost per lane: 4)
; CHECK: LV: Selecting VF: 4.
;
%5 = icmp eq i32 %3, 0
@@ -2281,16 +2186,7 @@ define hidden void @two_floats_same_op(ptr noundef readonly captures(none) %a, p
; CHECK: store ir<%mul> to index 0
; CHECK: store ir<%mul8> to index 1
; CHECK: Cost for VF 2: 29 (Estimated cost per lane: 14.5)
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx>
-; CHECK: ir<%0> = load from index 0
-; CHECK: ir<%2> = load from index 1
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx1>
-; CHECK: ir<%1> = load from index 0
-; CHECK: ir<%3> = load from index 1
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx3>
-; CHECK: store ir<%mul> to index 0
-; CHECK: store ir<%mul8> to index 1
-; CHECK: Cost for VF 4: 26 (Estimated cost per lane: 6.5)
+; CHECK: Cost for VF 4: 12 (Estimated cost per lane: 3)
; CHECK: LV: Selecting VF: 4.
;
entry:
@@ -2385,16 +2281,7 @@ define hidden void @two_bytes_two_floats_same_op(ptr noundef readonly captures(n
; CHECK: store ir<%mul> to index 0
; CHECK: store ir<%mul11> to index 1
; CHECK: Cost for VF 2: 52 (Estimated cost per lane: 26)
-; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx>
-; CHECK: ir<%0> = load from index 0
-; CHECK: ir<%2> = load from index 1
-; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx1>
-; CHECK: ir<%1> = load from index 0
-; CHECK: ir<%3> = load from index 1
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx4>
-; CHECK: store ir<%mul> to index 0
-; CHECK: store ir<%mul11> to index 1
-; CHECK: Cost for VF 4: 48 (Estimated cost per lane: 12)
+; CHECK: Cost for VF 4: 18 (Estimated cost per lane: 4.5)
; CHECK: LV: Selecting VF: 4.
;
entry:
@@ -2605,16 +2492,7 @@ define hidden void @two_shorts_two_floats_same_op(ptr noundef readonly captures(
; CHECK: store ir<%mul> to index 0
; CHECK: store ir<%mul11> to index 1
; CHECK: Cost for VF 2: 45 (Estimated cost per lane: 22.5)
-; CHECK: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx>
-; CHECK: ir<%0> = load from index 0
-; CHECK: ir<%2> = load from index 1
-; CHECK: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx1>
-; CHECK: ir<%1> = load from index 0
-; CHECK: ir<%3> = load from index 1
-; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%arrayidx4>
-; CHECK: store ir<%mul> to index 0
-; CHECK: store ir<%mul11> to index 1
-; CHECK: Cost for VF 4: 36 (Estimated cost per lane: 9)
+; CHECK: Cost for VF 4: 16 (Estimated cost per lane: 4)
; CHECK: LV: Selecting VF: 4.
;
entry:
@@ -2976,22 +2854,7 @@ define hidden void @four_bytes_four_floats_same_op(ptr noundef readonly captures
; CHECK: store ir<%mul19> to index 2
; CHECK: store ir<%mul27> to index 3
; CHECK: Cost for VF 2: 99 (Estimated cost per lane: 49.5)
-; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx>
-; CHECK: ir<%0> = load from index 0
-; CHECK: ir<%2> = load from index 1
-; CHECK: ir<%4> = load from index 2
-; CHECK: ir<%6> = load from index 3
-; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx1>
-; CHECK: ir<%1> = load from index 0
-; CHECK: ir<%3> = load from index 1
-; CHECK: ir<%5> = load from index 2
-; CHECK: ir<%7> = load from index 3
-; CHECK: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx4>
-; CHECK: store ir<%mul> to index 0
-; CHECK: store ir<%mul11> to index 1
-; CHECK: store ir<%mul19> to index 2
-; CHECK: store ir<%mul27> to index 3
-; CHECK: Cost for VF 4: 108 (Estimated cost per lane: 27)
+; CHECK: Cost for VF 4: 18 (Estimated cost per lane: 4.5)
; CHECK: LV: Selecting VF: 4.
;
entry:
@@ -3312,22 +3175,7 @@ define hidden void @four_shorts_four_floats_same_op(ptr noundef readonly capture
; CHECK: store ir<%mul19> to index 2
; CHECK: store ir<%mul27> to index 3
; CHECK: Cost for VF 2: 78 (Estimated cost per lane: 39)
-; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx>
-; CHECK: ir<%0> = load from index 0
-; CHECK: ir<%2> = load from index 1
-; CHECK: ir<%4> = load from index 2
-; CHECK: ir<%6> = load from index 3
-; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx1>
-; CHECK: ir<%1> = load from index 0
-; CHECK: ir<%3> = load from index 1
-; CHECK: ir<%5> = load from index 2
-; CHECK: ir<%7> = load from index 3
-; CHECK: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx4>
-; CHECK: store ir<%mul> to index 0
-; CHECK: store ir<%mul11> to index 1
-; CHECK: store ir<%mul19> to index 2
-; CHECK: store ir<%mul27> to index 3
-; CHECK: Cost for VF 4: 100 (Estimated cost per lane: 25)
+; CHECK: Cost for VF 4: 16 (Estimated cost per lane: 4)
; CHECK: LV: Selecting VF: 4.
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-half.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-half.ll
index 748cb00270ba4..888861fdc3eca 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-half.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/interleaved-load-half.ll
@@ -9,8 +9,6 @@ target triple = "i386-unknown-linux-gnu"
define void @stride8(half %k, i32 %width_) {
; CHECK-LABEL: 'stride8'
-; CHECK: Cost of 148 for VF 32: INTERLEAVE-GROUP with factor 8, ir<%arrayidx>
-;
entry:
@@ -98,8 +96,6 @@ for.body:
define void @stride3(half %k, i32 %width_) {
; CHECK-LABEL: 'stride3'
-; CHECK: Cost of 18 for VF 32: INTERLEAVE-GROUP with factor 3, ir<%arrayidx>
-;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
index c599c7e3e17e2..19b19bdf64753 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
@@ -223,30 +223,40 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF10:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT13:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX9]], 2
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[P]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD10]], [[WIDE_LOAD11]]
-; CHECK-NEXT: store <4 x double> [[TMP14]], ptr [[TMP13]], align 8
-; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX9]], 1
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT12]], 99
-; CHECK-NEXT: br i1 [[TMP15]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <2 x i32> <i32 1, i32 5>
+; CHECK-NEXT: [[STRIDED_VEC11:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <2 x i32> <i32 2, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <2 x i32> <i32 3, i32 7>
+; CHECK-NEXT: [[TMP15:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[STRIDED_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd <2 x double> [[STRIDED_VEC10]], [[STRIDED_VEC10]]
+; CHECK-NEXT: [[TMP17:%.*]] = fadd <2 x double> [[STRIDED_VEC11]], [[STRIDED_VEC11]]
+; CHECK-NEXT: [[TMP18:%.*]] = fadd <2 x double> [[STRIDED_VEC12]], [[STRIDED_VEC12]]
+; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP21]], <8 x double> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP13]], align 8
+; CHECK-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX9]], 2
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT13]], 98
+; CHECK-NEXT: br i1 [[TMP22]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi i64 [ 396, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 384, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL14:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL14:%.*]] = phi i64 [ 392, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 384, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL15:%.*]] = phi i64 [ 1, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[CNT_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL14]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL14]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[CNT_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[V0:%.*]] = load double, ptr [[GEP0]], align 8
; CHECK-NEXT: [[R0:%.*]] = fadd double [[V0]], [[V0]]
@@ -266,7 +276,7 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
; CHECK-NEXT: [[CNT_NEXT]] = add i64 [[CNT]], -1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[CNT_NEXT]], 0
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
index 3b1219be49ab6..3d3f20ae252a0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
@@ -149,49 +149,104 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[IV]], 6
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT5]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT7]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT9]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = shl nsw i64 [[TMP15]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = shl nsw i64 [[TMP11]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = shl nsw i64 [[TMP2]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
-; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP13]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP14]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP18]], align 8
+; CHECK-NEXT: [[TMP19:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP20:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT6]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP21:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT8]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP22:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT10]], [[WIDE_LOAD4]]
+; CHECK-NEXT: store <4 x i64> [[TMP19]], ptr [[TMP4]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP20]], ptr [[TMP16]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP21]], ptr [[TMP17]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP22]], ptr [[TMP18]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[N_MOD_VF11:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC12:%.*]] = sub i64 [[N]], [[N_MOD_VF11]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]
; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i64> poison, i64 [[L_FACTOR]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT15]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IV1]], 1
; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
-; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
-; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]
-; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[TMP0]], 1
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i64>, ptr [[DATA_0]], align 8
+; CHECK-NEXT: [[TMP28:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT16]], [[WIDE_LOAD14]]
+; CHECK-NEXT: store <4 x i64> [[TMP28]], ptr [[DATA_0]], align 8
+; CHECK-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[IV1]], 2
+; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC12]]
+; CHECK-NEXT: br i1 [[TMP29]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N18:%.*]] = icmp eq i64 [[N]], [[N_VEC12]]
+; CHECK-NEXT: br i1 [[CMP_N18]], label %[[EXIT]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC12]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV2]]
+; CHECK-NEXT: [[L_FACTOR1:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
+; CHECK-NEXT: [[TMP30:%.*]] = shl nsw i64 [[IV2]], 1
+; CHECK-NEXT: [[DATA_2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP30]]
+; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_2]], align 8
+; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR1]], [[L_0]]
+; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_2]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[TMP30]], 1
; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8
-; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]
+; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR1]], [[L_1]]
; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV2]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -246,7 +301,7 @@ define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr no
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -269,7 +324,7 @@ define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr no
; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -324,7 +379,7 @@ define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %fa
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -347,7 +402,7 @@ define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %fa
; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_1]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -402,7 +457,7 @@ define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %fa
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -425,7 +480,7 @@ define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %fa
; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_0]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -486,7 +541,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP16]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -510,7 +565,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -570,7 +625,7 @@ define void @test_3xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -595,7 +650,7 @@ define void @test_3xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: store i64 [[MUL_2]], ptr [[DATA_2]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -704,7 +759,7 @@ define void @test_3xi32(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -728,7 +783,7 @@ define void @test_3xi32(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: store i32 [[MUL_2]], ptr [[DATA_2]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -812,7 +867,7 @@ define i64 @test_4xi64_induction_live_out(ptr noalias %data, ptr noalias %factor
; CHECK-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP14]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -834,7 +889,7 @@ define i64 @test_4xi64_induction_live_out(ptr noalias %data, ptr noalias %factor
; CHECK-NEXT: store <4 x i64> [[TMP23]], ptr [[TMP22]], align 8
; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX11]], 1
; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N]]
-; CHECK-NEXT: br i1 [[TMP24]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
@@ -862,7 +917,7 @@ define i64 @test_4xi64_induction_live_out(ptr noalias %data, ptr noalias %factor
; CHECK-NEXT: store i64 [[MUL_3]], ptr [[DATA_3]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_NEXT]], %[[LOOP]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[IV_NEXT_LCSSA]]
@@ -905,20 +960,21 @@ exit:
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
-; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]}
-; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META1]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META2]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META2]], [[META1]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META1]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META1]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]], [[META2]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META2]], [[META1]]}
; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]], [[META2]]}
-; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META2]], [[META1]]}
+; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]], [[META2]]}
+; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
index e9dfebff0d809..8d51bb66df3d2 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
@@ -179,7 +179,7 @@ define void @noalias_metadata_from_versioning(ptr %base, ptr %end, ptr %src) {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
@@ -188,15 +188,10 @@ define void @noalias_metadata_from_versioning(ptr %base, ptr %end, ptr %src) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x float> [[STRIDED_VEC]], splat (float 1.000000e+01)
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = load <2 x float>, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x float> [[STRIDED_VEC3]], splat (float 1.000000e+01)
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: store <2 x float> [[TMP7]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/pr31098.ll b/llvm/test/Transforms/LoopVectorize/pr31098.ll
index f183e30a04d29..9466250b5bf17 100644
--- a/llvm/test/Transforms/LoopVectorize/pr31098.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr31098.ll
@@ -56,7 +56,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; CHECK-LABEL: Test
; CHECK: LAA: No unsafe dependent memory operations in loop. We don't need runtime memory checks.
; CHECK: vector.body:
-; CHECK: <4 x i32>
+; CHECK: <4 x float>
%class.Complex = type { float, float }
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll
index 5d2a9c20874aa..cf1b30b3bd5ba 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll
@@ -17,13 +17,20 @@ define void @add4(ptr noalias noundef %x, ptr noalias noundef %y, i32 noundef %n
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC24:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = add <32 x i16> [[WIDE_VEC24]], [[WIDE_VEC]]
-; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i16> [[WIDE_LOAD22]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i16> [[WIDE_LOAD23]], [[WIDE_LOAD21]]
+; CHECK-NEXT: store <4 x i16> [[TMP6]], ptr [[TMP1]], align 2
+; CHECK-NEXT: store <4 x i16> [[TMP7]], ptr [[TMP5]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: for.end:
@@ -389,16 +396,26 @@ define void @addmul(ptr noalias noundef %x, ptr noundef %y, ptr noundef %z, i32
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Z:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC31:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y]], i64 [[TMP1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[WIDE_LOAD28:%.*]] = load <4 x i16>, ptr [[TMP3]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC36:%.*]] = load <32 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT: [[TMP4:%.*]] = mul <32 x i16> [[WIDE_VEC31]], [[WIDE_VEC]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = add <32 x i16> [[TMP4]], [[WIDE_VEC36]]
-; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X]], i64 [[TMP1]]
+; CHECK-NEXT: [[WIDE_LOAD29:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[WIDE_LOAD30:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i16> [[WIDE_LOAD29]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i16> [[WIDE_LOAD30]], [[WIDE_LOAD28]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X1:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X1]], i64 [[TMP1]]
+; CHECK-NEXT: [[WIDE_LOAD31:%.*]] = load <4 x i16>, ptr [[TMP8]], align 2
+; CHECK-NEXT: [[WIDE_LOAD32:%.*]] = load <4 x i16>, ptr [[TMP9]], align 2
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i16> [[TMP6]], [[WIDE_LOAD31]]
+; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i16> [[TMP7]], [[WIDE_LOAD32]]
+; CHECK-NEXT: store <4 x i16> [[TMP10]], ptr [[TMP8]], align 2
+; CHECK-NEXT: store <4 x i16> [[TMP11]], ptr [[TMP9]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: for.end:
More information about the llvm-commits
mailing list