[llvm] [AArch64] Improve load / store costs for non-power2 vector types. (PR #214695)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 03:18:41 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/214695
This adjusts the existing non-power-2 load/store cost routines to handle larger than 128bit vectors, by splitting out the initial 128bit chunks and costing the remainder with loads + inserts.
Fixes #214475
>From 2751a06c032a2f2abbb73be341d213099b2c93e6 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Fri, 7 Aug 2026 11:18:00 +0100
Subject: [PATCH] [AArch64] Improve load / store costs for non-power2 vector
types.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 33 +-
.../Analysis/CostModel/AArch64/load_store.ll | 344 +++++++++---------
.../Analysis/CostModel/AArch64/vec3-ops.ll | 64 ++--
.../masked-loads-side-effects-after-vec.ll | 33 +-
.../VectorCombine/AArch64/shrink-types.ll | 14 +
5 files changed, 265 insertions(+), 223 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a380cca93d84d..7bfdf1a9c3ceb 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5619,7 +5619,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
EVT EltVT = VT.getVectorElementType();
unsigned EltSize = EltVT.getScalarSizeInBits();
if (!isPowerOf2_32(EltSize) || EltSize < 8 || EltSize > 64 ||
- VT.getVectorNumElements() >= (128 / EltSize) || Alignment != Align(1))
+ Alignment != Align(1))
return LT.first;
// FIXME: v3i8 lowering currently is very inefficient, due to automatic
// widening to v4i8, which produces suboptimal results.
@@ -5629,22 +5629,23 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
// Check non-power-of-2 loads/stores for legal vector element types with
// NEON. Non-power-of-2 memory ops will get broken down to a set of
// operations on smaller power-of-2 ops, including ld1/st1.
- LLVMContext &C = Ty->getContext();
- InstructionCost Cost(0);
- SmallVector<EVT> TypeWorklist;
- TypeWorklist.push_back(VT);
- while (!TypeWorklist.empty()) {
- EVT CurrVT = TypeWorklist.pop_back_val();
- unsigned CurrNumElements = CurrVT.getVectorNumElements();
- if (isPowerOf2_32(CurrNumElements)) {
- Cost += 1;
- continue;
- }
+ InstructionCost Cost = VT.getVectorNumElements() / (128 / EltSize);
+ unsigned Remainder = VT.getVectorNumElements() % (128 / EltSize);
+ if (Remainder != 0) {
+ SmallVector<std::pair<unsigned, unsigned>> TypeWorklist;
+ TypeWorklist.push_back({Remainder, 0});
+ while (!TypeWorklist.empty()) {
+ auto [CurrNumElements, Offset] = TypeWorklist.pop_back_val();
+ if (isPowerOf2_32(CurrNumElements)) {
+ // 1 per load/store + possible lane insert.
+ Cost += 1 + (Offset == 0 ? 0 : 1);
+ continue;
+ }
- unsigned PrevPow2 = NextPowerOf2(CurrNumElements) / 2;
- TypeWorklist.push_back(EVT::getVectorVT(C, EltVT, PrevPow2));
- TypeWorklist.push_back(
- EVT::getVectorVT(C, EltVT, CurrNumElements - PrevPow2));
+ unsigned PrevPow2 = NextPowerOf2(CurrNumElements) / 2;
+ TypeWorklist.push_back({PrevPow2, Offset});
+ TypeWorklist.push_back({CurrNumElements - PrevPow2, Offset + PrevPow2});
+ }
}
return Cost;
}
diff --git a/llvm/test/Analysis/CostModel/AArch64/load_store.ll b/llvm/test/Analysis/CostModel/AArch64/load_store.ll
index 08db76e588cbb..e9db76f7eede6 100644
--- a/llvm/test/Analysis/CostModel/AArch64/load_store.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/load_store.ll
@@ -626,130 +626,130 @@ define i32 @stores_partial_align1(i32 %arg) {
; CHECK-LABEL: 'stores_partial_align1'
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <1 x i64> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <3 x i64> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <5 x i64> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <6 x i64> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <5 x i64> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <6 x i64> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <7 x i64> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <1 x double> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <3 x double> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <5 x double> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <6 x double> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <5 x double> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <6 x double> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <7 x double> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <1 x i32> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <2 x i32> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i32> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i32> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <5 x i32> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <6 x i32> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <7 x i32> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <9 x i32> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <10 x i32> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <11 x i32> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <12 x i32> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <7 x i32> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <9 x i32> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <10 x i32> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <11 x i32> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <12 x i32> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <13 x i32> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <14 x i32> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <15 x i32> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <15 x i32> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <1 x float> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <2 x float> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x float> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x float> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <5 x float> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <6 x float> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <7 x float> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <9 x float> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <10 x float> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <11 x float> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <12 x float> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <7 x float> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <9 x float> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <10 x float> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <11 x float> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <12 x float> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <13 x float> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <14 x float> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <15 x float> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <15 x float> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <1 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: store <2 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <4 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <9 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <10 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <11 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <11 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <12 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <13 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <14 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <15 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <17 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <18 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <19 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <20 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <21 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <22 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <23 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <24 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <13 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <14 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:1 SizeLat:2 for: store <15 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <17 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <18 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <19 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <20 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <21 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <22 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:1 SizeLat:4 for: store <23 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <24 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <25 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <26 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <27 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <27 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <28 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <29 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <30 x i16> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <31 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <29 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <30 x i16> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:1 SizeLat:4 for: store <31 x i16> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <1 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: store <2 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <4 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <9 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <10 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <11 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <12 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <13 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <14 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: store <15 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <9 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <10 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <11 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <12 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <13 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <14 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: store <15 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <17 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <18 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <19 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <19 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <20 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <21 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <22 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <23 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <21 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <22 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:1 SizeLat:2 for: store <23 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <24 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <25 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <26 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <27 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <28 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <29 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <30 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <31 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <33 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <34 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <35 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <36 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <37 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <38 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <39 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <40 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <41 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <42 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <43 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <44 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <45 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <46 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <47 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <48 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <25 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <26 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:1 SizeLat:2 for: store <27 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:1 SizeLat:2 for: store <28 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:1 SizeLat:2 for: store <29 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:1 SizeLat:2 for: store <30 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:2 Lat:1 SizeLat:2 for: store <31 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <33 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <34 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <35 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <36 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <37 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <38 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:1 SizeLat:4 for: store <39 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <40 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <41 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <42 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:1 SizeLat:4 for: store <43 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:1 SizeLat:4 for: store <44 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:1 SizeLat:4 for: store <45 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:1 SizeLat:4 for: store <46 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:1 SizeLat:4 for: store <47 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:1 SizeLat:4 for: store <48 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <49 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <50 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <51 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <51 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <52 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <53 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <54 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <55 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <53 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <54 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:1 SizeLat:4 for: store <55 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <56 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <57 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <58 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <59 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <60 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <61 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <62 x i8> undef, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:1 SizeLat:4 for: store <63 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <57 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <58 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:1 SizeLat:4 for: store <59 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:1 SizeLat:4 for: store <60 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:1 SizeLat:4 for: store <61 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:1 SizeLat:4 for: store <62 x i8> undef, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:1 SizeLat:4 for: store <63 x i8> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i1> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i1> undef, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i1> undef, ptr undef, align 1
@@ -2123,130 +2123,130 @@ define i32 @loads_partial_align1(i32 %arg) {
; CHECK-LABEL: 'loads_partial_align1'
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %1 = load <1 x i64>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %2 = load <3 x i64>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %3 = load <5 x i64>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %4 = load <6 x i64>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %3 = load <5 x i64>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %4 = load <6 x i64>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %5 = load <7 x i64>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %6 = load <1 x double>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %7 = load <3 x double>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %8 = load <5 x double>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %9 = load <6 x double>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %8 = load <5 x double>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %9 = load <6 x double>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %10 = load <7 x double>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %11 = load <1 x i32>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %12 = load <2 x i32>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %13 = load <3 x i32>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %13 = load <3 x i32>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %14 = load <5 x i32>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %15 = load <6 x i32>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %16 = load <7 x i32>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %17 = load <9 x i32>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %18 = load <10 x i32>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %19 = load <11 x i32>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %20 = load <12 x i32>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %16 = load <7 x i32>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %17 = load <9 x i32>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %18 = load <10 x i32>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %19 = load <11 x i32>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %20 = load <12 x i32>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %21 = load <13 x i32>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %22 = load <14 x i32>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %23 = load <15 x i32>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %23 = load <15 x i32>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %24 = load <1 x float>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %25 = load <2 x float>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %26 = load <3 x float>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %26 = load <3 x float>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %27 = load <5 x float>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %28 = load <6 x float>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %29 = load <7 x float>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %30 = load <9 x float>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %31 = load <10 x float>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %32 = load <11 x float>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %33 = load <12 x float>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %29 = load <7 x float>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %30 = load <9 x float>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %31 = load <10 x float>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %32 = load <11 x float>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %33 = load <12 x float>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %34 = load <13 x float>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %35 = load <14 x float>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %36 = load <15 x float>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %36 = load <15 x float>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %37 = load <1 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %38 = load <2 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %39 = load <3 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %39 = load <3 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %40 = load <4 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %41 = load <5 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %42 = load <6 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %43 = load <7 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %41 = load <5 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %42 = load <6 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %43 = load <7 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %44 = load <9 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %45 = load <10 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %46 = load <11 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %46 = load <11 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %47 = load <12 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %48 = load <13 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %49 = load <14 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %50 = load <15 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %51 = load <17 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %52 = load <18 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %53 = load <19 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %54 = load <20 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %55 = load <21 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %56 = load <22 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %57 = load <23 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %58 = load <24 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %48 = load <13 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %49 = load <14 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:5 SizeLat:2 for: %50 = load <15 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %51 = load <17 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %52 = load <18 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %53 = load <19 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %54 = load <20 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %55 = load <21 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %56 = load <22 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:4 for: %57 = load <23 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %58 = load <24 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %59 = load <25 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %60 = load <26 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %61 = load <27 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %61 = load <27 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %62 = load <28 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %63 = load <29 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %64 = load <30 x i16>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %65 = load <31 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %63 = load <29 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %64 = load <30 x i16>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:7 SizeLat:4 for: %65 = load <31 x i16>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %66 = load <1 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %67 = load <2 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:4 SizeLat:1 for: %68 = load <3 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %69 = load <4 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %70 = load <5 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %71 = load <6 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %72 = load <7 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %70 = load <5 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %71 = load <6 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %72 = load <7 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %73 = load <8 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %74 = load <9 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %75 = load <10 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %76 = load <11 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %77 = load <12 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %78 = load <13 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %79 = load <14 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %80 = load <15 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %74 = load <9 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %75 = load <10 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %76 = load <11 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %77 = load <12 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %78 = load <13 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %79 = load <14 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:1 Lat:4 SizeLat:1 for: %80 = load <15 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %81 = load <17 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %82 = load <18 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %83 = load <19 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %83 = load <19 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %84 = load <20 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %85 = load <21 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %86 = load <22 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %87 = load <23 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %85 = load <21 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %86 = load <22 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:5 SizeLat:2 for: %87 = load <23 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %88 = load <24 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %89 = load <25 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %90 = load <26 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %91 = load <27 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %92 = load <28 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %93 = load <29 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %94 = load <30 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %95 = load <31 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %96 = load <33 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %97 = load <34 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %98 = load <35 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %99 = load <36 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %100 = load <37 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %101 = load <38 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %102 = load <39 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %103 = load <40 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %104 = load <41 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %105 = load <42 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %106 = load <43 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %107 = load <44 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %108 = load <45 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %109 = load <46 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %110 = load <47 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %111 = load <48 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %89 = load <25 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %90 = load <26 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:5 SizeLat:2 for: %91 = load <27 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:5 SizeLat:2 for: %92 = load <28 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:5 SizeLat:2 for: %93 = load <29 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:5 SizeLat:2 for: %94 = load <30 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:2 Lat:5 SizeLat:2 for: %95 = load <31 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %96 = load <33 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %97 = load <34 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %98 = load <35 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %99 = load <36 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %100 = load <37 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %101 = load <38 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:4 for: %102 = load <39 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %103 = load <40 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %104 = load <41 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %105 = load <42 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:4 for: %106 = load <43 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:7 SizeLat:4 for: %107 = load <44 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:4 for: %108 = load <45 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:4 for: %109 = load <46 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:7 SizeLat:4 for: %110 = load <47 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:7 SizeLat:4 for: %111 = load <48 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %112 = load <49 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %113 = load <50 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %114 = load <51 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %114 = load <51 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %115 = load <52 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %116 = load <53 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %117 = load <54 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %118 = load <55 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %116 = load <53 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %117 = load <54 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:7 SizeLat:4 for: %118 = load <55 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %119 = load <56 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %120 = load <57 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %121 = load <58 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %122 = load <59 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %123 = load <60 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %124 = load <61 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %125 = load <62 x i8>, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:7 SizeLat:4 for: %126 = load <63 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %120 = load <57 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %121 = load <58 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:7 SizeLat:4 for: %122 = load <59 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:7 SizeLat:4 for: %123 = load <60 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:7 SizeLat:4 for: %124 = load <61 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:7 SizeLat:4 for: %125 = load <62 x i8>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:7 SizeLat:4 for: %126 = load <63 x i8>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:4 SizeLat:1 for: %127 = load <3 x i1>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:1 Lat:4 SizeLat:1 for: %128 = load <5 x i1>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:1 Lat:4 SizeLat:1 for: %129 = load <6 x i1>, ptr undef, align 1
diff --git a/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll b/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
index f234341b0fd86..282e536d9bb63 100644
--- a/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
@@ -3,12 +3,12 @@
define void @vec3_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i32>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i32>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i32> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %cmp = icmp uge <3 x i32> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i32> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sel = select <3 x i1> %cmp, <3 x i32> %add, <3 x i32> %sub
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i32> %sel, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i32> %sel, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <3 x i32>, ptr %src, align 1
@@ -41,12 +41,12 @@ define void @vec3_i32_default_alignment(<3 x i32> %a, <3 x i32> %b, ptr %src, pt
define void @vec3_i16(<3 x i16> %a, <3 x i16> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i16> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %cmp = icmp uge <3 x i16> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i16> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sel = select <3 x i1> %cmp, <3 x i16> %add, <3 x i16> %sub
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i16> %sel, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i16> %sel, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <3 x i16>, ptr %src, align 1
@@ -60,9 +60,9 @@ define void @vec3_i16(<3 x i16> %a, <3 x i16> %b, ptr %src, ptr %dst) {
define void @vec7_i16(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec7_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <7 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <7 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <7 x i16> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i16> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i16> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <7 x i16>, ptr %src, align 1
@@ -73,9 +73,9 @@ define void @vec7_i16(ptr %src, ptr %dst) {
define void @vec6_i16(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec6_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <6 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <6 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <6 x i16> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i16> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i16> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <6 x i16>, ptr %src, align 1
@@ -86,9 +86,9 @@ define void @vec6_i16(ptr %src, ptr %dst) {
define void @vec5_i16(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec5_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <5 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <5 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <5 x i16> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i16> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i16> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <5 x i16>, ptr %src, align 1
@@ -99,12 +99,12 @@ define void @vec5_i16(ptr %src, ptr %dst) {
define void @vec3_i16_zext_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i16_zext_i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %l.ext = zext <3 x i16> %l to <3 x i32>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i32> %l.ext, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i32> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub.trunc = trunc <3 x i32> %sub to <3 x i16>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i16> %sub.trunc, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i16> %sub.trunc, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <3 x i16>, ptr %src, align 1
@@ -190,10 +190,10 @@ define void @vec3_i30(<3 x i30> %a, <3 x i30> %b, ptr %src, ptr %dst) {
define void @vec3_float(<3 x float> %a, <3 x float> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_float'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x float>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x float>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %add = fadd <3 x float> %l, %b
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %sub = fsub <3 x float> %add, %a
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x float> %sub, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x float> %sub, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <3 x float>, ptr %src, align 1
@@ -205,10 +205,10 @@ define void @vec3_float(<3 x float> %a, <3 x float> %b, ptr %src, ptr %dst) {
define void @vec3_half(<3 x half> %a, <3 x half> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_half'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x half>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x half>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %add = fadd <3 x half> %l, %b
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %sub = fsub <3 x half> %add, %a
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x half> %sub, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x half> %sub, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <3 x half>, ptr %src, align 1
@@ -220,9 +220,9 @@ define void @vec3_half(<3 x half> %a, <3 x half> %b, ptr %src, ptr %dst) {
define void @vec15_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec15_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <15 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <15 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <15 x i8> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: store <15 x i8> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: store <15 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <15 x i8>, ptr %src, align 1
@@ -233,9 +233,9 @@ define void @vec15_i8(ptr %src, ptr %dst) {
define void @vec14_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec14_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <14 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <14 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <14 x i8> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <14 x i8> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <14 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <14 x i8>, ptr %src, align 1
@@ -246,9 +246,9 @@ define void @vec14_i8(ptr %src, ptr %dst) {
define void @vec13_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec13_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <13 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <13 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <13 x i8> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <13 x i8> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <13 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <13 x i8>, ptr %src, align 1
@@ -259,9 +259,9 @@ define void @vec13_i8(ptr %src, ptr %dst) {
define void @vec12_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec12_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <12 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <12 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <12 x i8> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <12 x i8> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <12 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <12 x i8>, ptr %src, align 1
@@ -272,9 +272,9 @@ define void @vec12_i8(ptr %src, ptr %dst) {
define void @vec11_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec11_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <11 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <11 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <11 x i8> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <11 x i8> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <11 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <11 x i8>, ptr %src, align 1
@@ -285,10 +285,10 @@ define void @vec11_i8(ptr %src, ptr %dst) {
define void @vec7_i8(<7 x i8> %a, <7 x i8> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec7_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <7 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <7 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <7 x i8> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <7 x i8> %add, %a
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i8> %sub, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i8> %sub, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <7 x i8>, ptr %src, align 1
@@ -300,9 +300,9 @@ define void @vec7_i8(<7 x i8> %a, <7 x i8> %b, ptr %src, ptr %dst) {
define void @vec6_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec6_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <6 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <6 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <6 x i8> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i8> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <6 x i8>, ptr %src, align 1
@@ -313,9 +313,9 @@ define void @vec6_i8(ptr %src, ptr %dst) {
define void @vec5_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec5_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <5 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <5 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <5 x i8> %l, %l
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i8> %add, ptr %dst, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%l = load <5 x i8>, ptr %src, align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll
index df4c527aa216a..fefcfd3a9a567 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll
@@ -8,9 +8,36 @@ define void @test() {
; CHECK-LABEL: define void @test() {
; CHECK-NEXT: [[TMP1:%.*]] = call dereferenceable_or_null(16) ptr @malloc()
; CHECK-NEXT: [[TMP2:%.*]] = load volatile ptr, ptr null, align 8
-; CHECK-NEXT: [[TMP3:%.*]] = load <15 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <15 x i8> [[TMP3]], <15 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: store <8 x i8> [[TMP4]], ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP2]], i64 1
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP1]], i64 4
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP2]], i64 2
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[TMP9]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP1]], i64 6
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP2]], i64 3
+; CHECK-NEXT: store i8 [[TMP11]], ptr [[TMP12]], align 1
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP1]], i64 8
+; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP13]], align 1
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[TMP2]], i64 4
+; CHECK-NEXT: store i8 [[TMP14]], ptr [[TMP15]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP1]], i64 10
+; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[TMP2]], i64 5
+; CHECK-NEXT: store i8 [[TMP17]], ptr [[TMP18]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP1]], i64 12
+; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP2]], i64 6
+; CHECK-NEXT: store i8 [[TMP20]], ptr [[TMP21]], align 1
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP1]], i64 14
+; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i64 7
+; CHECK-NEXT: store i8 [[TMP23]], ptr [[TMP24]], align 1
; CHECK-NEXT: ret void
;
%1 = call dereferenceable_or_null(16) ptr @malloc()
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll b/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll
index 6c0ab8b9abaff..c9a4df6a7e2d8 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll
@@ -108,4 +108,18 @@ define <2 x i32> @pr108698(<2 x i64> %x, <2 x i32> %y) {
ret <2 x i32> %lshr
}
+define void @stride_load_31(ptr %src, ptr %dst) {
+; CHECK-LABEL: @stride_load_31(
+; CHECK-NEXT: [[V:%.*]] = load <32 x i8>, ptr [[SRC:%.*]], align 1
+; CHECK-NEXT: [[STRIDED:%.*]] = shufflevector <32 x i8> [[V]], <32 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT: store <16 x i8> [[STRIDED]], ptr [[DST:%.*]], align 1
+; CHECK-NEXT: ret void
+;
+ %v = load <32 x i8>, ptr %src, align 1
+ %strided = shufflevector <32 x i8> %v, <32 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+ store <16 x i8> %strided, ptr %dst, align 1
+ ret void
+}
+
+
declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
More information about the llvm-commits
mailing list